diff --git a/README.md b/README.md new file mode 100644 index 000000000..704cf40ee --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +# Сдача проектной работы 9 спринта + +## Задание 1. Повышение безопасности системы + +1. [диаграмма](https://drive.google.com/file/d/1bP4TGxW8FGkh9_jxoARsXJKgMDPVWP05/view?usp=sharing, "диаграмма") + +## Задача 2. Улучшите безопасность существующего приложения, заменив Code Grant на PKCE + +Clickhouse слишком хлопотно, как и выгрузка за конкретные даты, нужно интерфейс писать. Реализовал OLAP на PostgreSQL + +1. [диаграмма]("https://drive.google.com/file/d/1X3OlEfDEQUajJNN5xkmB_01UY0uVSaeD/view?usp=sharing", "диаграмма") +2. Код Airflow в папке проекта **airflow** +3. API в папке **backend** +4. user1 не имеет доступа, получит ошибку **403: Insufficient role**. prothetic1 имеет доступ и получит свой отчёт +5. [UI кнопка для отчёта](https://disk.yandex.ru/i/-kQhnAhK2RM9VA, "UI кнопка") + +## Как запустить +1. UP docker-compose.yaml +2. Отклываем Airflow UI http://localhost:8081/home, убеждается, что пайплайны подняты: **init_data** и **olap_pipeline** +3. **init_data** срабатывает единожды +4. **olap_pipeline** срабатывает каждые 5 минут +5. http://localhost:3000/ логинимся под **prothetic[1..3]**, нажимаем кнопку **Download Report** +6. Выгружаются только от **prothetic[1..3]** \ No newline at end of file diff --git a/airflow/Dockerfile b/airflow/Dockerfile new file mode 100644 index 000000000..afacce351 --- /dev/null +++ b/airflow/Dockerfile @@ -0,0 +1,6 @@ +FROM apache/airflow:2.9.1-python3.9 +USER root + +COPY ./airflow/requirements.txt . +RUN pip3 install --upgrade pip +RUN pip3 install --no-cache-dir -r requirements.txt \ No newline at end of file diff --git a/airflow/dags/__pycache__/dag_olap.cpython-39.pyc b/airflow/dags/__pycache__/dag_olap.cpython-39.pyc new file mode 100644 index 000000000..7b22c2048 Binary files /dev/null and b/airflow/dags/__pycache__/dag_olap.cpython-39.pyc differ diff --git a/airflow/dags/__pycache__/dag_sample.cpython-39.pyc b/airflow/dags/__pycache__/dag_sample.cpython-39.pyc new file mode 100644 index 000000000..fa8578bcb Binary files /dev/null and b/airflow/dags/__pycache__/dag_sample.cpython-39.pyc differ diff --git a/airflow/dags/dag_olap.py b/airflow/dags/dag_olap.py new file mode 100644 index 000000000..6f94dfe14 --- /dev/null +++ b/airflow/dags/dag_olap.py @@ -0,0 +1,178 @@ +from airflow import DAG +from airflow.operators.python import PythonOperator +from airflow.providers.postgres.hooks.postgres import PostgresHook +from airflow.providers.postgres.operators.postgres import PostgresOperator +from datetime import datetime, timedelta +import logging +import pandas as pd + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': datetime(2024, 1, 1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=1), +} + +def extract_data(): + try: + pg_hook = PostgresHook(postgres_conn_id='write_to_pg') + + query = """ + SELECT + c.user_id, + c.user_name, + c.email, + c.prosthesis_model, + c.prosthesis_serial, + c.installation_date, + t.usage_date, + COUNT(t.session_count) as total_sessions, + SUM(t.total_usage_minutes) as total_usage_minutes, + AVG(t.max_force_application) as avg_max_force, + AVG(t.avg_battery_level) as avg_battery_level, + COUNT(t.error_codes) as total_error_count, + MAX(t.last_active) as last_active + FROM crm_table c + LEFT JOIN telemetry_table t ON c.user_id = t.user_id + GROUP BY + c.user_id, c.user_name, c.email, c.prosthesis_model, + c.prosthesis_serial, c.installation_date, t.usage_date + """ + + connection = pg_hook.get_conn() + cursor = connection.cursor() + cursor.execute(query) + + columns = [desc[0] for desc in cursor.description] + results = cursor.fetchall() + + cursor.close() + connection.close() + + return pd.DataFrame(results, columns=columns) + + except Exception as e: + logging.error(f"Error extracting data: {str(e)}") + raise + +def transform_data(**kwargs): + ti = kwargs['ti'] + df = ti.xcom_pull(task_ids='extract_data') + + if df.empty: + logging.info("No data to transform") + return pd.DataFrame() + + df['usage_date'] = pd.to_datetime(df['usage_date']) + df['installation_date'] = pd.to_datetime(df['installation_date']) + df['last_active'] = pd.to_datetime(df['last_active']) + + df.fillna({ + 'total_sessions': 0, + 'total_usage_minutes': 0, + 'avg_max_force': 0, + 'avg_battery_level': 0, + 'total_error_count': 0 + }, inplace=True) + + return df + +def load_to_olap(**kwargs): + ti = kwargs['ti'] + df = ti.xcom_pull(task_ids='transform_data') + + if df.empty: + logging.info("No data to load") + return + + try: + pg_hook = PostgresHook(postgres_conn_id='write_to_pg') + connection = pg_hook.get_conn() + cursor = connection.cursor() + + insert_sql = """ + INSERT INTO olap_table ( + user_id, user_name, email, prosthesis_model, prosthesis_serial, + installation_date, usage_date, total_sessions, total_usage_minutes, + avg_max_force, avg_battery_level, total_error_count, last_active + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + """ + + records = [] + for _, row in df.iterrows(): + record = ( + row['user_id'], row['user_name'], row['email'], + row['prosthesis_model'], row['prosthesis_serial'], + row['installation_date'], row['usage_date'], + row['total_sessions'], row['total_usage_minutes'], + row['avg_max_force'], row['avg_battery_level'], + row['total_error_count'], row['last_active'] + ) + records.append(record) + + cursor.executemany(insert_sql, records) + connection.commit() + + logging.info(f"Successfully loaded {len(records)} records to OLAP table") + + cursor.close() + connection.close() + + except Exception as e: + logging.error(f"Error loading data to OLAP: {str(e)}") + raise + +with DAG('olap_pipeline', + default_args=default_args, + schedule_interval='*/5 * * * *', + tags=['olap', 'prosthesis', 'etl'], + catchup=False) as dag: + + create_olap_table = PostgresOperator( + task_id='create_olap_table', + postgres_conn_id='write_to_pg', + sql=""" + DROP TABLE IF EXISTS olap_table; + CREATE TABLE olap_table ( + id SERIAL PRIMARY KEY, + user_id UUID NOT NULL, + user_name VARCHAR(100) NOT NULL, + email VARCHAR(150) NOT NULL, + prosthesis_model VARCHAR(100) NOT NULL, + prosthesis_serial VARCHAR(50) NOT NULL, + installation_date DATE NOT NULL, + usage_date DATE NOT NULL, + total_sessions INTEGER NOT NULL DEFAULT 0, + total_usage_minutes INTEGER NOT NULL DEFAULT 0, + avg_max_force FLOAT NOT NULL DEFAULT 0, + avg_battery_level FLOAT NOT NULL DEFAULT 0, + total_error_count INTEGER NOT NULL DEFAULT 0, + last_active TIMESTAMP NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + """ + ) + + extract_task = PythonOperator( + task_id='extract_data', + python_callable=extract_data, + dag=dag, + ) + + transform_task = PythonOperator( + task_id='transform_data', + python_callable=transform_data, + dag=dag, + ) + + load_task = PythonOperator( + task_id='load_to_olap', + python_callable=load_to_olap, + dag=dag, + ) + + create_olap_table>>extract_task >> transform_task >> load_task \ No newline at end of file diff --git a/airflow/dags/dag_sample.py b/airflow/dags/dag_sample.py new file mode 100644 index 000000000..cade6ed04 --- /dev/null +++ b/airflow/dags/dag_sample.py @@ -0,0 +1,116 @@ +from airflow import DAG +from airflow.operators.python import PythonOperator +from airflow.providers.postgres.operators.postgres import PostgresOperator +from datetime import datetime +import csv + +default_args = { + 'owner': 'airflow', + 'start_date': datetime(2024, 12, 1), +} + +def generate_crm(): + CSV_FILE_PATH = './dags/data/crm_sample.csv' + with open( CSV_FILE_PATH, 'r') as csvfile: + csvreader = csv.reader(csvfile, delimiter=";") + + insert_queries = [] + is_header = True + for row in csvreader: + if is_header: + is_header = False + continue + insert_query = f"INSERT INTO crm_table (id,user_id,user_name,email,prosthesis_model,prosthesis_serial,installation_date) VALUES ({row[0]}, '{row[1]}', '{row[2]}','{row[3]}','{row[4]}','{row[5]}','{row[6]}');" + insert_queries.append(insert_query) + + with open('./dags/sql/insert_crm.sql', 'w') as f: + for query in insert_queries: + f.write(f"{query}\n") + +def generate_telemetry(): + CSV_FILE_PATH = './dags/data/telemetry_sample.csv' + with open( CSV_FILE_PATH, 'r') as csvfile: + csvreader = csv.reader(csvfile, delimiter=";") + + insert_queries = [] + is_header = True + for row in csvreader: + if is_header: + is_header = False + continue + insert_query = f"INSERT INTO telemetry_table (id,user_id,usage_date,session_count,total_usage_minutes,max_force_application,avg_battery_level,error_codes,last_active) VALUES ({row[0]}, '{row[1]}', '{row[2]}',{row[3]},{row[4]},{row[5]},{row[6]},'{row[7]}','{row[8]}');" + insert_queries.append(insert_query) + + with open('./dags/sql/insert_telemetry.sql', 'w') as f: + for query in insert_queries: + f.write(f"{query}\n") + +with DAG('init_data', + default_args=default_args, + schedule_interval='@once', + tags=['init', 'etl'], + catchup=False) as dag: + + create_crm_table = PostgresOperator( + task_id='create_crm_table', + postgres_conn_id='write_to_pg', + sql=""" + DROP TABLE IF EXISTS crm_table; + CREATE TABLE crm_table ( + id SERIAL PRIMARY KEY, + user_id UUID NOT NULL, + user_name VARCHAR(100) NOT NULL, + email VARCHAR(150) NOT NULL, + prosthesis_model VARCHAR(100) NOT NULL, + prosthesis_serial VARCHAR(50) NOT NULL UNIQUE, + installation_date DATE NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + """ + ) + + create_telemetry_table = PostgresOperator( + task_id='create_telemetry_table', + postgres_conn_id='write_to_pg', + sql=""" + DROP TABLE IF EXISTS telemetry_table; + CREATE TABLE telemetry_table ( + id SERIAL PRIMARY KEY, + user_id UUID NOT NULL, + usage_date DATE NOT NULL, + session_count INTEGER NOT NULL DEFAULT 0, + total_usage_minutes INTEGER NOT NULL DEFAULT 0, + max_force_application FLOAT NOT NULL DEFAULT 0, + avg_battery_level FLOAT NOT NULL DEFAULT 0, + error_codes JSONB DEFAULT '[]'::jsonb, + last_active TIMESTAMP NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + """ + ) + + generate_crm_queries = PythonOperator( + task_id='generate_crm_queries', + python_callable=generate_crm + ) + + generate_telemetry_queries = PythonOperator( + task_id='generate_telemetry_queries', + python_callable=generate_telemetry + ) + + run_insert_crm_queries = PostgresOperator( + task_id='run_insert_crm_queries', + postgres_conn_id='write_to_pg', + sql='sql/insert_crm.sql' + ) + + run_insert_telemetry_queries = PostgresOperator( + task_id='run_insert_telemetry_queries', + postgres_conn_id='write_to_pg', + sql='sql/insert_telemetry.sql' + ) + + create_crm_table>>create_telemetry_table>>generate_crm_queries>>generate_telemetry_queries>>run_insert_crm_queries>>run_insert_telemetry_queries \ No newline at end of file diff --git a/airflow/dags/data/crm_sample.csv b/airflow/dags/data/crm_sample.csv new file mode 100644 index 000000000..c78e0a0b5 --- /dev/null +++ b/airflow/dags/data/crm_sample.csv @@ -0,0 +1,11 @@ +id;user_id;customer_name;customer_email;prosthesis_model;prosthesis_serial;installation_date +1;728fb50b-9774-4433-adb8-117133edf7ed;prothetic1;prothetic1@example.com;AlphaLimb Pro;SN-ALP2023-001;2023-05-15 +2;0cda865e-05ea-40a1-8392-a3eef7e677bb;prothetic2;prothetic2@example.com;BioHand Plus;SN-BHP2023-045;2023-06-20 +3;5e21faf7-f526-4c89-af6f-23562aed7e12;prothetic3;prothetic3@example.com;SmartGrip Ultra;SN-SGU2023-178;2023-07-10 +4;728fb50b-9774-4433-adb8-117133edf7ed;prothetic1;prothetic1@example.com;AlphaLimb Pro v2;SN-ALP2024-002;2024-01-12 +5;0cda865e-05ea-40a1-8392-a3eef7e677bb;prothetic2;prothetic2@example.com;BioHand Elite;SN-BHE2024-087;2024-02-18 +6;5e21faf7-f526-4c89-af6f-23562aed7e12;prothetic3;prothetic3@example.com;SmartGrip Pro;SN-SGP2024-192;2024-03-22 +7;728fb50b-9774-4433-adb8-117133edf7ed;prothetic1;prothetic1@example.com;AlphaLimb Pro;SN-ALP2023-003;2023-08-15 +8;0cda865e-05ea-40a1-8392-a3eef7e677bb;prothetic2;prothetic2@example.com;BioHand Plus;SN-BHP2023-051;2023-09-30 +9;5e21faf7-f526-4c89-af6f-23562aed7e12;prothetic3;prothetic3@example.com;SmartGrip Ultra;SN-SGU2023-181;2023-10-14 +10;728fb50b-9774-4433-adb8-117133edf7ed;prothetic1;prothetic1@example.com;AlphaLimb Pro v2;SN-ALP2024-004;2024-04-01 \ No newline at end of file diff --git a/airflow/dags/data/telemetry_sample.csv b/airflow/dags/data/telemetry_sample.csv new file mode 100644 index 000000000..eb206fd53 --- /dev/null +++ b/airflow/dags/data/telemetry_sample.csv @@ -0,0 +1,11 @@ +id;user_id;usage_date;session_count;total_usage_minutes;max_force_application;avg_battery_level;error_codes;last_active +1;728fb50b-9774-4433-adb8-117133edf7ed;2024-12-01;8;320;12.5;87.2;"[""ERR-001"",""ERR-003""]";2024-12-01 18:45:32 +2;0cda865e-05ea-40a1-8392-a3eef7e677bb;2024-12-01;12;480;15.8;92.1;"[""ERR-002""]";2024-12-01 19:30:15 +3;5e21faf7-f526-4c89-af6f-23562aed7e12;2024-12-01;6;210;9.3;95.4;"[]";2024-12-01 17:20:48 +4;728fb50b-9774-4433-adb8-117133edf7ed;2024-12-02;10;380;11.2;89.7;"[""ERR-001""]";2024-12-02 19:15:22 +5;0cda865e-05ea-40a1-8392-a3eef7e677bb;2024-12-02;15;550;16.5;90.8;"[""ERR-002"",""ERR-004""]";2024-12-02 20:05:37 +6;5e21faf7-f526-4c89-af6f-23562aed7e12;2024-12-02;7;260;10.1;93.2;"[]";2024-12-02 18:40:19 +7;728fb50b-9774-4433-adb8-117133edf7ed;2024-12-03;9;340;13.1;88.5;"[""ERR-003""]";2024-12-03 18:30:45 +8;0cda865e-05ea-40a1-8392-a3eef7e677bb;2024-12-03;11;420;14.9;91.3;"[""ERR-005""]";2024-12-03 19:45:28 +9;5e21faf7-f526-4c89-af6f-23562aed7e12;2024-12-03;5;190;8.7;94.8;"[]";2024-12-03 16:55:12 +10;728fb50b-9774-4433-adb8-117133edf7ed;2024-12-04;7;270;10.5;86.4;"[""ERR-001"",""ERR-006""]";2024-12-04 17:45:18 \ No newline at end of file diff --git a/airflow/dags/sql/insert_crm.sql b/airflow/dags/sql/insert_crm.sql new file mode 100644 index 000000000..a871746dc --- /dev/null +++ b/airflow/dags/sql/insert_crm.sql @@ -0,0 +1,10 @@ +INSERT INTO crm_table (id,user_id,user_name,email,prosthesis_model,prosthesis_serial,installation_date) VALUES (1, '728fb50b-9774-4433-adb8-117133edf7ed', 'prothetic1','prothetic1@example.com','AlphaLimb Pro','SN-ALP2023-001','2023-05-15'); +INSERT INTO crm_table (id,user_id,user_name,email,prosthesis_model,prosthesis_serial,installation_date) VALUES (2, '0cda865e-05ea-40a1-8392-a3eef7e677bb', 'prothetic2','prothetic2@example.com','BioHand Plus','SN-BHP2023-045','2023-06-20'); +INSERT INTO crm_table (id,user_id,user_name,email,prosthesis_model,prosthesis_serial,installation_date) VALUES (3, '5e21faf7-f526-4c89-af6f-23562aed7e12', 'prothetic3','prothetic3@example.com','SmartGrip Ultra','SN-SGU2023-178','2023-07-10'); +INSERT INTO crm_table (id,user_id,user_name,email,prosthesis_model,prosthesis_serial,installation_date) VALUES (4, '728fb50b-9774-4433-adb8-117133edf7ed', 'prothetic1','prothetic1@example.com','AlphaLimb Pro v2','SN-ALP2024-002','2024-01-12'); +INSERT INTO crm_table (id,user_id,user_name,email,prosthesis_model,prosthesis_serial,installation_date) VALUES (5, '0cda865e-05ea-40a1-8392-a3eef7e677bb', 'prothetic2','prothetic2@example.com','BioHand Elite','SN-BHE2024-087','2024-02-18'); +INSERT INTO crm_table (id,user_id,user_name,email,prosthesis_model,prosthesis_serial,installation_date) VALUES (6, '5e21faf7-f526-4c89-af6f-23562aed7e12', 'prothetic3','prothetic3@example.com','SmartGrip Pro','SN-SGP2024-192','2024-03-22'); +INSERT INTO crm_table (id,user_id,user_name,email,prosthesis_model,prosthesis_serial,installation_date) VALUES (7, '728fb50b-9774-4433-adb8-117133edf7ed', 'prothetic1','prothetic1@example.com','AlphaLimb Pro','SN-ALP2023-003','2023-08-15'); +INSERT INTO crm_table (id,user_id,user_name,email,prosthesis_model,prosthesis_serial,installation_date) VALUES (8, '0cda865e-05ea-40a1-8392-a3eef7e677bb', 'prothetic2','prothetic2@example.com','BioHand Plus','SN-BHP2023-051','2023-09-30'); +INSERT INTO crm_table (id,user_id,user_name,email,prosthesis_model,prosthesis_serial,installation_date) VALUES (9, '5e21faf7-f526-4c89-af6f-23562aed7e12', 'prothetic3','prothetic3@example.com','SmartGrip Ultra','SN-SGU2023-181','2023-10-14'); +INSERT INTO crm_table (id,user_id,user_name,email,prosthesis_model,prosthesis_serial,installation_date) VALUES (10, '728fb50b-9774-4433-adb8-117133edf7ed', 'prothetic1','prothetic1@example.com','AlphaLimb Pro v2','SN-ALP2024-004','2024-04-01'); diff --git a/airflow/dags/sql/insert_telemetry.sql b/airflow/dags/sql/insert_telemetry.sql new file mode 100644 index 000000000..3cfa211e3 --- /dev/null +++ b/airflow/dags/sql/insert_telemetry.sql @@ -0,0 +1,10 @@ +INSERT INTO telemetry_table (id,user_id,usage_date,session_count,total_usage_minutes,max_force_application,avg_battery_level,error_codes,last_active) VALUES (1, '728fb50b-9774-4433-adb8-117133edf7ed', '2024-12-01',8,320,12.5,87.2,'["ERR-001","ERR-003"]','2024-12-01 18:45:32'); +INSERT INTO telemetry_table (id,user_id,usage_date,session_count,total_usage_minutes,max_force_application,avg_battery_level,error_codes,last_active) VALUES (2, '0cda865e-05ea-40a1-8392-a3eef7e677bb', '2024-12-01',12,480,15.8,92.1,'["ERR-002"]','2024-12-01 19:30:15'); +INSERT INTO telemetry_table (id,user_id,usage_date,session_count,total_usage_minutes,max_force_application,avg_battery_level,error_codes,last_active) VALUES (3, '5e21faf7-f526-4c89-af6f-23562aed7e12', '2024-12-01',6,210,9.3,95.4,'[]','2024-12-01 17:20:48'); +INSERT INTO telemetry_table (id,user_id,usage_date,session_count,total_usage_minutes,max_force_application,avg_battery_level,error_codes,last_active) VALUES (4, '728fb50b-9774-4433-adb8-117133edf7ed', '2024-12-02',10,380,11.2,89.7,'["ERR-001"]','2024-12-02 19:15:22'); +INSERT INTO telemetry_table (id,user_id,usage_date,session_count,total_usage_minutes,max_force_application,avg_battery_level,error_codes,last_active) VALUES (5, '0cda865e-05ea-40a1-8392-a3eef7e677bb', '2024-12-02',15,550,16.5,90.8,'["ERR-002","ERR-004"]','2024-12-02 20:05:37'); +INSERT INTO telemetry_table (id,user_id,usage_date,session_count,total_usage_minutes,max_force_application,avg_battery_level,error_codes,last_active) VALUES (6, '5e21faf7-f526-4c89-af6f-23562aed7e12', '2024-12-02',7,260,10.1,93.2,'[]','2024-12-02 18:40:19'); +INSERT INTO telemetry_table (id,user_id,usage_date,session_count,total_usage_minutes,max_force_application,avg_battery_level,error_codes,last_active) VALUES (7, '728fb50b-9774-4433-adb8-117133edf7ed', '2024-12-03',9,340,13.1,88.5,'["ERR-003"]','2024-12-03 18:30:45'); +INSERT INTO telemetry_table (id,user_id,usage_date,session_count,total_usage_minutes,max_force_application,avg_battery_level,error_codes,last_active) VALUES (8, '0cda865e-05ea-40a1-8392-a3eef7e677bb', '2024-12-03',11,420,14.9,91.3,'["ERR-005"]','2024-12-03 19:45:28'); +INSERT INTO telemetry_table (id,user_id,usage_date,session_count,total_usage_minutes,max_force_application,avg_battery_level,error_codes,last_active) VALUES (9, '5e21faf7-f526-4c89-af6f-23562aed7e12', '2024-12-03',5,190,8.7,94.8,'[]','2024-12-03 16:55:12'); +INSERT INTO telemetry_table (id,user_id,usage_date,session_count,total_usage_minutes,max_force_application,avg_battery_level,error_codes,last_active) VALUES (10, '728fb50b-9774-4433-adb8-117133edf7ed', '2024-12-04',7,270,10.5,86.4,'["ERR-001","ERR-006"]','2024-12-04 17:45:18'); diff --git a/airflow/db/init-db.sql b/airflow/db/init-db.sql new file mode 100644 index 000000000..544816668 --- /dev/null +++ b/airflow/db/init-db.sql @@ -0,0 +1,2 @@ +CREATE DATABASE sample; +GRANT ALL PRIVILEGES ON DATABASE sample TO airflow; \ No newline at end of file diff --git a/airflow/docker-compose.yaml b/airflow/docker-compose.yaml new file mode 100644 index 000000000..b90a02924 --- /dev/null +++ b/airflow/docker-compose.yaml @@ -0,0 +1,137 @@ +x-airflow-common: &airflow-common + build: + context: . + dockerfile: Dockerfile + environment: &airflow-common-env + AIRFLOW__CORE__EXECUTOR: LocalExecutor #локальный запуск + AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow + AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow + AIRFLOW__CORE__FERNET_KEY: '' + AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' + AIRFLOW__CORE__LOAD_EXAMPLES: 'false' + AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session' + AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true' + AIRFLOW__WEBSERVER__SECRET_KEY: 'your_airflow_webserver_sec_key' + _PIP_ADDITIONAL_REQUIREMENTS: '' + AIRFLOW_INPUT_DIR: '/opt/airflow/dag-inputs' + volumes: + - ./dags:/opt/airflow/dags + - ./dags/sql:/opt/airflow/dags/sql + - ./requirements.txt:/opt/airflow/requirements.txt + - ./data:/opt/airflow/sample_files + user: "${AIRFLOW_UID:-50000}:0" + depends_on: &airflow-common-depends-on + postgres: + condition: service_healthy + networks: + - dag_sample + +services: + postgres: + image: postgres:16.0 + volumes: + - ./db/init-db.sql:/docker-entrypoint-initdb.d/db.sql + environment: + POSTGRES_USER: airflow + POSTGRES_PASSWORD: airflow + POSTGRES_DB: airflow + logging: + options: + max-size: 10m + max-file: "3" + healthcheck: + test: + - CMD + - pg_isready + - -U + - airflow + interval: 10s + retries: 5 + start_period: 5s + restart: always + networks: + - dag_sample + + airflow-webserver: + <<: *airflow-common + + ports: + - "8080:8080" + depends_on: + postgres: + condition: service_healthy + airflow-init: + condition: service_completed_successfully + command: webserver + networks: + - dag_sample + healthcheck: + test: [ "CMD-SHELL", "curl -f http://localhost:8080/health || exit 1" ] + interval: 30s + timeout: 10s + retries: 3 + + airflow-scheduler: + <<: *airflow-common + networks: + - dag_sample + command: scheduler + depends_on: + postgres: + condition: service_healthy + airflow-webserver: + condition: service_healthy + healthcheck: + test: [ "CMD-SHELL", "curl -f http://localhost:8080/health || exit 1" ] + interval: 30s + timeout: 10s + retries: 3 + + airflow-triggerer: + <<: *airflow-common + depends_on: + postgres: + condition: service_healthy + airflow-init: + condition: service_completed_successfully + networks: + - dag_sample + command: bash -c "airflow triggerer" + healthcheck: + test: + - CMD-SHELL + - airflow jobs check --job-type TriggererJob --hostname "${HOSTNAME}" + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + + airflow-cli: + <<: *airflow-common + depends_on: + postgres: + condition: service_healthy + networks: + - dag_sample + profiles: + - debug + command: + - bash + - -c + - airflow + + airflow-init: + <<: *airflow-common + depends_on: + postgres: + condition: service_healthy + command: > + bash -c " airflow db init && airflow users create \ --username admin \ --firstname admin \ --lastname admin \ --role Admin \ --email admin@sample.ru \ --password admin + + " + networks: + - dag_sample + +networks: + dag_sample: \ No newline at end of file diff --git a/airflow/requirements.txt b/airflow/requirements.txt new file mode 100644 index 000000000..86ed6d0bb --- /dev/null +++ b/airflow/requirements.txt @@ -0,0 +1,2 @@ +psycopg2-binary +pandas>=1.3.0 \ No newline at end of file diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 000000000..63c7d781f --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.12 +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY main.py . +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file diff --git a/backend/main.py b/backend/main.py new file mode 100644 index 000000000..e0092a57e --- /dev/null +++ b/backend/main.py @@ -0,0 +1,105 @@ +from fastapi import FastAPI, Depends, HTTPException, status +from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +from fastapi.middleware.cors import CORSMiddleware +from jose import jwt, jwk, JWTError +import requests +import os +import uuid +import hashlib +from typing import Dict +from random import randint +import logging +import psycopg2 +from psycopg2.extras import RealDictCursor + +app = FastAPI() + +origins = [ + "http://localhost:3000" +] + +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["authorization"], +) + +bearer_scheme = HTTPBearer() + + +KEYCLOAK_URL = os.environ.get("KEYCLOAK_URL", "http://keycloak:8080") +KEYCLOAK_REALM = os.environ.get("KEYCLOAK_REALM", "reports-realm") + +KEYCLOAK_ISSUER = "http://localhost:8080/realms/reports-realm" +KEYCLOAK_CERTS_URL = f"{KEYCLOAK_URL}/realms/{KEYCLOAK_REALM}/protocol/openid-connect/certs" + + +def get_certs(): + response = requests.get(KEYCLOAK_CERTS_URL) + if response.status_code != 200: + raise HTTPException(status_code=500, detail="Failed to fetch Keycloak public keys") + return response.json() + +def get_current_user(token: HTTPAuthorizationCredentials = Depends(bearer_scheme)): + try: + unverified_header = jwt.get_unverified_header(token.credentials) + kid = unverified_header.get("kid") + + keys = get_certs()["keys"] + key = next((k for k in keys if k["kid"] == kid), None) + if not key: + raise Exception("Public key not found") + + public_key = jwk.construct(key) + + payload = jwt.decode( + token.credentials, + public_key, + algorithms=[key["alg"]], + audience="reports-api", + issuer=KEYCLOAK_ISSUER + ) + + logging.warning(payload) + + roles = payload.get("realm_access", {}).get("roles", []) + if "prothetic_user" not in roles: + raise HTTPException(status_code=403, detail="Insufficient role") + + return payload + + except JWTError as e: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid token") + + +@app.get("/reports") +def get_reports(user: Dict = Depends(get_current_user)): + user_id = user.get("sub", "unknown") + logging.warning(user_id) + + db_config = { + 'host': os.environ.get("AIRFLOW_DB_HOST", "postgres"), + 'database': os.environ.get("AIRFLOW_DB_NAME", "sample"), + 'user': os.environ.get("AIRFLOW_DB_USER", "airflow"), + 'password': os.environ.get("AIRFLOW_DB_PASSWORD", "airflow"), + 'port': '5432' + } + + connection = psycopg2.connect(**db_config) + cursor = connection.cursor(cursor_factory=RealDictCursor) + + query = """ + SELECT * FROM olap_table + WHERE user_id = %s + """ + params = [user_id] + + cursor.execute(query, params) + reports = [dict(row) for row in cursor.fetchall()] + + cursor.close() + connection.close() + + return {"reports": reports} \ No newline at end of file diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 000000000..0fcb87735 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,5 @@ +fastapi +uvicorn +python-jose[cryptography] +requests +psycopg2 \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index f21d8cf27..2cea12a29 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,6 +1,35 @@ version: '3.8' +x-airflow-common: &airflow-common + build: + context: . + dockerfile: ./airflow/Dockerfile + environment: &airflow-common-env + AIRFLOW__CORE__EXECUTOR: LocalExecutor + AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow + AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow + AIRFLOW__CORE__FERNET_KEY: '' + AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' + AIRFLOW__CORE__LOAD_EXAMPLES: 'false' + AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session' + AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true' + AIRFLOW__WEBSERVER__SECRET_KEY: 'your_airflow_webserver_sec_key' + _PIP_ADDITIONAL_REQUIREMENTS: '' + AIRFLOW_INPUT_DIR: '/opt/airflow/dag-inputs' + volumes: + - ./airflow/dags:/opt/airflow/dags + - ./airflow/dags/sql:/opt/airflow/dags/sql + - ./airflow/requirements.txt:/opt/airflow/requirements.txt + - ./airflow/data:/opt/airflow/sample_files + user: "${AIRFLOW_UID:-50000}:0" + depends_on: &airflow-common-depends-on + postgres: + condition: service_healthy + networks: + - dag_sample + services: + # Keycloak services keycloak_db: image: postgres:14 environment: @@ -11,6 +40,9 @@ services: - ./postgres-keycloak-data:/var/lib/postgresql/data ports: - "5433:5432" + networks: + - app-network + keycloak: image: quay.io/keycloak/keycloak:21.1 environment: @@ -20,6 +52,7 @@ services: KC_DB_URL: jdbc:postgresql://keycloak_db:5432/keycloak_db KC_DB_USERNAME: keycloak_user KC_DB_PASSWORD: keycloak_password + KEYCLOAK_FRONTEND_URL: http://keycloak:8080 command: - start-dev - --import-realm @@ -29,6 +62,10 @@ services: - "8080:8080" depends_on: - keycloak_db + networks: + - app-network + + # Frontend service frontend: build: context: ./frontend @@ -36,7 +73,139 @@ services: ports: - "3000:3000" environment: - REACT_APP_API_URL: http://localhost:8000 - REACT_APP_KEYCLOAK_URL: http://localhost:8080 + REACT_APP_API_URL: http://backend:8000 + REACT_APP_KEYCLOAK_URL: http://keycloak:8080 REACT_APP_KEYCLOAK_REALM: reports-realm REACT_APP_KEYCLOAK_CLIENT_ID: reports-frontend + networks: + - app-network + + backend: + build: + context: ./backend + dockerfile: Dockerfile + ports: + - "8000:8000" + environment: + KEYCLOAK_URL: http://keycloak:8080 + KEYCLOAK_REALM: reports-realm + AIRFLOW_DB_HOST: postgres + AIRFLOW_DB_PORT: 5432 + AIRFLOW_DB_NAME: sample + AIRFLOW_DB_USER: airflow + AIRFLOW_DB_PASSWORD: airflow + depends_on: + - keycloak + - postgres + networks: + - app-network + - dag_sample + + # Airflow services + postgres: + image: postgres:16.0 + volumes: + - ./airflow/db/init-db.sql:/docker-entrypoint-initdb.d/db.sql + environment: + POSTGRES_USER: airflow + POSTGRES_PASSWORD: airflow + POSTGRES_DB: airflow + logging: + options: + max-size: 10m + max-file: "3" + healthcheck: + test: + - CMD + - pg_isready + - -U + - airflow + interval: 10s + retries: 5 + start_period: 5s + restart: always + networks: + - dag_sample + - app-network + + airflow-webserver: + <<: *airflow-common + ports: + - "8081:8080" + depends_on: + postgres: + condition: service_healthy + airflow-init: + condition: service_completed_successfully + command: webserver + networks: + - dag_sample + healthcheck: + disable: true + + airflow-scheduler: + <<: *airflow-common + networks: + - dag_sample + command: scheduler + depends_on: + postgres: + condition: service_healthy + healthcheck: + test: [ "CMD-SHELL", "curl -f http://localhost:8081/health || exit 1" ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + + airflow-triggerer: + <<: *airflow-common + depends_on: + postgres: + condition: service_healthy + airflow-init: + condition: service_completed_successfully + networks: + - dag_sample + command: bash -c "airflow triggerer" + healthcheck: + test: + - CMD-SHELL + - airflow jobs check --job-type TriggererJob --hostname "${HOSTNAME}" + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + + airflow-cli: + <<: *airflow-common + depends_on: + postgres: + condition: service_healthy + networks: + - dag_sample + profiles: + - debug + command: + - bash + - -c + - airflow + + airflow-init: + <<: *airflow-common + depends_on: + postgres: + condition: service_healthy + command: > + bash -c " airflow db init && airflow users create \ --username admin \ --firstname admin \ --lastname admin \ --role Admin \ --email admin@sample.ru \ --password admin + + " + networks: + - dag_sample + +networks: + dag_sample: + driver: bridge + app-network: + driver: bridge \ No newline at end of file diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 7340f0c2f..9a4a0911f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -13,11 +13,14 @@ "@types/react": "^18.0.0", "@types/react-dom": "^18.0.0", "keycloak-js": "^21.1.0", - "react": "^18.2.0", - "react-dom": "^18.2.0", + "react": "^18.3.1", + "react-dom": "^18.3.1", "react-scripts": "5.0.1", "tailwindcss": "^3.3.0", "typescript": "^4.9.5" + }, + "devDependencies": { + "tsx": "^4.20.5" } }, "node_modules/@alloc/quick-lru": { @@ -2226,6 +2229,448 @@ "postcss-selector-parser": "^6.0.10" } }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.9.tgz", + "integrity": "sha512-OaGtL73Jck6pBKjNIe24BnFE6agGl+6KxDtTfHhy1HmhthfKouEcOhqpSL64K4/0WCtbKFLOdzD/44cJ4k9opA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.9.tgz", + "integrity": "sha512-5WNI1DaMtxQ7t7B6xa572XMXpHAaI/9Hnhk8lcxF4zVN4xstUgTlvuGDorBguKEnZO70qwEcLpfifMLoxiPqHQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.9.tgz", + "integrity": "sha512-IDrddSmpSv51ftWslJMvl3Q2ZT98fUSL2/rlUXuVqRXHCs5EUF1/f+jbjF5+NG9UffUDMCiTyh8iec7u8RlTLg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.9.tgz", + "integrity": "sha512-I853iMZ1hWZdNllhVZKm34f4wErd4lMyeV7BLzEExGEIZYsOzqDWDf+y082izYUE8gtJnYHdeDpN/6tUdwvfiw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.9.tgz", + "integrity": "sha512-XIpIDMAjOELi/9PB30vEbVMs3GV1v2zkkPnuyRRURbhqjyzIINwj+nbQATh4H9GxUgH1kFsEyQMxwiLFKUS6Rg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.9.tgz", + "integrity": "sha512-jhHfBzjYTA1IQu8VyrjCX4ApJDnH+ez+IYVEoJHeqJm9VhG9Dh2BYaJritkYK3vMaXrf7Ogr/0MQ8/MeIefsPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.9.tgz", + "integrity": "sha512-z93DmbnY6fX9+KdD4Ue/H6sYs+bhFQJNCPZsi4XWJoYblUqT06MQUdBCpcSfuiN72AbqeBFu5LVQTjfXDE2A6Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.9.tgz", + "integrity": "sha512-mrKX6H/vOyo5v71YfXWJxLVxgy1kyt1MQaD8wZJgJfG4gq4DpQGpgTB74e5yBeQdyMTbgxp0YtNj7NuHN0PoZg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.9.tgz", + "integrity": "sha512-HBU2Xv78SMgaydBmdor38lg8YDnFKSARg1Q6AT0/y2ezUAKiZvc211RDFHlEZRFNRVhcMamiToo7bDx3VEOYQw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.9.tgz", + "integrity": "sha512-BlB7bIcLT3G26urh5Dmse7fiLmLXnRlopw4s8DalgZ8ef79Jj4aUcYbk90g8iCa2467HX8SAIidbL7gsqXHdRw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.9.tgz", + "integrity": "sha512-e7S3MOJPZGp2QW6AK6+Ly81rC7oOSerQ+P8L0ta4FhVi+/j/v2yZzx5CqqDaWjtPFfYz21Vi1S0auHrap3Ma3A==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.9.tgz", + "integrity": "sha512-Sbe10Bnn0oUAB2AalYztvGcK+o6YFFA/9829PhOCUS9vkJElXGdphz0A3DbMdP8gmKkqPmPcMJmJOrI3VYB1JQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.9.tgz", + "integrity": "sha512-YcM5br0mVyZw2jcQeLIkhWtKPeVfAerES5PvOzaDxVtIyZ2NUBZKNLjC5z3/fUlDgT6w89VsxP2qzNipOaaDyA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.9.tgz", + "integrity": "sha512-++0HQvasdo20JytyDpFvQtNrEsAgNG2CY1CLMwGXfFTKGBGQT3bOeLSYE2l1fYdvML5KUuwn9Z8L1EWe2tzs1w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.9.tgz", + "integrity": "sha512-uNIBa279Y3fkjV+2cUjx36xkx7eSjb8IvnL01eXUKXez/CBHNRw5ekCGMPM0BcmqBxBcdgUWuUXmVWwm4CH9kg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.9.tgz", + "integrity": "sha512-Mfiphvp3MjC/lctb+7D287Xw1DGzqJPb/J2aHHcHxflUo+8tmN/6d4k6I2yFR7BVo5/g7x2Monq4+Yew0EHRIA==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.9.tgz", + "integrity": "sha512-iSwByxzRe48YVkmpbgoxVzn76BXjlYFXC7NvLYq+b+kDjyyk30J0JY47DIn8z1MO3K0oSl9fZoRmZPQI4Hklzg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.9.tgz", + "integrity": "sha512-9jNJl6FqaUG+COdQMjSCGW4QiMHH88xWbvZ+kRVblZsWrkXlABuGdFJ1E9L7HK+T0Yqd4akKNa/lO0+jDxQD4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.9.tgz", + "integrity": "sha512-RLLdkflmqRG8KanPGOU7Rpg829ZHu8nFy5Pqdi9U01VYtG9Y0zOG6Vr2z4/S+/3zIyOxiK6cCeYNWOFR9QP87g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.9.tgz", + "integrity": "sha512-YaFBlPGeDasft5IIM+CQAhJAqS3St3nJzDEgsgFixcfZeyGPCd6eJBWzke5piZuZ7CtL656eOSYKk4Ls2C0FRQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.9.tgz", + "integrity": "sha512-1MkgTCuvMGWuqVtAvkpkXFmtL8XhWy+j4jaSO2wxfJtilVCi0ZE37b8uOdMItIHz4I6z1bWWtEX4CJwcKYLcuA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.9.tgz", + "integrity": "sha512-4Xd0xNiMVXKh6Fa7HEJQbrpP3m3DDn43jKxMjxLLRjWnRsfxjORYJlXPO4JNcXtOyfajXorRKY9NkOpTHptErg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.9.tgz", + "integrity": "sha512-WjH4s6hzo00nNezhp3wFIAfmGZ8U7KtrJNlFMRKxiI9mxEK1scOMAaa9i4crUtu+tBr+0IN6JCuAcSBJZfnphw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.9.tgz", + "integrity": "sha512-mGFrVJHmZiRqmP8xFOc6b84/7xa5y5YvR1x8djzXpJBSv/UsNK6aqec+6JDjConTgvvQefdGhFDAs2DLAds6gQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.9.tgz", + "integrity": "sha512-b33gLVU2k11nVx1OhX3C8QQP6UHQK4ZtN56oFWvVXvz2VkDoe6fbG8TOgHFxEvqeqohmRnIHe5A1+HADk4OQww==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.9.tgz", + "integrity": "sha512-PPOl1mi6lpLNQxnGoyAfschAodRFYXJ+9fs6WHXz7CSWKbOqiMZsubC+BQsVKuul+3vKLuwTHsS2c2y9EoKwxQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.1.tgz", @@ -6470,6 +6915,48 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/esbuild": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.9.tgz", + "integrity": "sha512-CRbODhYyQx3qp7ZEwzxOk4JBqmD/seJrzPa/cGjY1VtIn5E09Oi9/dB4JwctnfZ8Q8iT7rioVv5k/FNT/uf54g==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.9", + "@esbuild/android-arm": "0.25.9", + "@esbuild/android-arm64": "0.25.9", + "@esbuild/android-x64": "0.25.9", + "@esbuild/darwin-arm64": "0.25.9", + "@esbuild/darwin-x64": "0.25.9", + "@esbuild/freebsd-arm64": "0.25.9", + "@esbuild/freebsd-x64": "0.25.9", + "@esbuild/linux-arm": "0.25.9", + "@esbuild/linux-arm64": "0.25.9", + "@esbuild/linux-ia32": "0.25.9", + "@esbuild/linux-loong64": "0.25.9", + "@esbuild/linux-mips64el": "0.25.9", + "@esbuild/linux-ppc64": "0.25.9", + "@esbuild/linux-riscv64": "0.25.9", + "@esbuild/linux-s390x": "0.25.9", + "@esbuild/linux-x64": "0.25.9", + "@esbuild/netbsd-arm64": "0.25.9", + "@esbuild/netbsd-x64": "0.25.9", + "@esbuild/openbsd-arm64": "0.25.9", + "@esbuild/openbsd-x64": "0.25.9", + "@esbuild/openharmony-arm64": "0.25.9", + "@esbuild/sunos-x64": "0.25.9", + "@esbuild/win32-arm64": "0.25.9", + "@esbuild/win32-ia32": "0.25.9", + "@esbuild/win32-x64": "0.25.9" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -7801,6 +8288,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-tsconfig": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.10.1.tgz", + "integrity": "sha512-auHyJ4AgMz7vgS8Hp3N6HXSmlMdUyhSUrfBF16w153rxtLIEOE+HGqaBppczZvnHLqQJfiHotCYpNhl0lUROFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -12516,6 +13016,7 @@ "version": "18.3.1", "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "license": "MIT", "dependencies": { "loose-envify": "^1.1.0" }, @@ -12647,6 +13148,7 @@ "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "license": "MIT", "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -12972,6 +13474,16 @@ "node": ">=8" } }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, "node_modules/resolve-url-loader": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-url-loader/-/resolve-url-loader-4.0.0.tgz", @@ -14665,6 +15177,26 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, + "node_modules/tsx": { + "version": "4.20.5", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.5.tgz", + "integrity": "sha512-+wKjMNU9w/EaQayHXb7WA7ZaHY6hN8WgfvHNQ3t1PnU91/7O8TcTnIhCDYTZwnt8JsO9IBqZ30Ln1r7pPF52Aw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.25.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index f2da19422..94de66e4b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -3,39 +3,42 @@ "version": "0.1.0", "private": true, "dependencies": { - "@react-keycloak/web": "^3.4.0", - "@types/node": "^16.18.0", - "@types/react": "^18.0.0", - "@types/react-dom": "^18.0.0", - "keycloak-js": "^21.1.0", - "react": "^18.2.0", - "react-dom": "^18.2.0", - "react-scripts": "5.0.1", - "tailwindcss": "^3.3.0", - "typescript": "^4.9.5" + "@react-keycloak/web": "^3.4.0", + "@types/node": "^16.18.0", + "@types/react": "^18.0.0", + "@types/react-dom": "^18.0.0", + "keycloak-js": "^21.1.0", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "react-scripts": "5.0.1", + "tailwindcss": "^3.3.0", + "typescript": "^4.9.5" }, "scripts": { - "start": "react-scripts start", - "build": "react-scripts build", - "test": "react-scripts test", - "eject": "react-scripts eject" + "start": "react-scripts start", + "build": "react-scripts build", + "test": "react-scripts test", + "eject": "react-scripts eject" }, "eslintConfig": { - "extends": [ - "react-app", - "react-app/jest" - ] + "extends": [ + "react-app", + "react-app/jest" + ] }, "browserslist": { - "production": [ - ">0.2%", - "not dead", - "not op_mini all" - ], - "development": [ - "last 1 chrome version", - "last 1 firefox version", - "last 1 safari version" - ] + "production": [ + ">0.2%", + "not dead", + "not op_mini all" + ], + "development": [ + "last 1 chrome version", + "last 1 firefox version", + "last 1 safari version" + ] + }, + "devDependencies": { + "tsx": "^4.20.5" } - } \ No newline at end of file +} diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index c5aaaf0e3..8c570ff39 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,6 +1,6 @@ import React from 'react'; import { ReactKeycloakProvider } from '@react-keycloak/web'; -import Keycloak, { KeycloakConfig } from 'keycloak-js'; +import Keycloak, { KeycloakConfig, KeycloakInitOptions } from 'keycloak-js'; import ReportPage from './components/ReportPage'; const keycloakConfig: KeycloakConfig = { @@ -11,9 +11,14 @@ const keycloakConfig: KeycloakConfig = { const keycloak = new Keycloak(keycloakConfig); +const initOptions:KeycloakInitOptions = { + onLoad: 'login-required', + pkceMethod: 'S256' +} + const App: React.FC = () => { return ( - +
diff --git a/frontend/src/components/ReportPage.tsx b/frontend/src/components/ReportPage.tsx index 63ec99fff..6241b42a6 100644 --- a/frontend/src/components/ReportPage.tsx +++ b/frontend/src/components/ReportPage.tsx @@ -5,6 +5,7 @@ const ReportPage: React.FC = () => { const { keycloak, initialized } = useKeycloak(); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); + const [reports, setReports] = useState(null); const downloadReport = async () => { if (!keycloak?.token) { @@ -16,12 +17,31 @@ const ReportPage: React.FC = () => { setLoading(true); setError(null); + await keycloak.updateToken(10) + const response = await fetch(`${process.env.REACT_APP_API_URL}/reports`, { headers: { 'Authorization': `Bearer ${keycloak.token}` } }); + console.log(await keycloak.loadUserInfo()) + + if (!response.ok) { + let message = `Error ${response.status}`; + try { + const data = await response.json(); + message += `: ${data.detail || JSON.stringify(data)}`; + } catch { + const text = await response.text(); + message += `: ${text}`; + } + throw new Error(message); + } + + const data = await response.json(); + setReports(data.reports || []); + } catch (err) { setError(err instanceof Error ? err.message : 'An error occurred'); @@ -48,7 +68,7 @@ const ReportPage: React.FC = () => { } return ( -
+

Usage Reports

@@ -67,6 +87,60 @@ const ReportPage: React.FC = () => { {error}
)} + + {reports && ( + + + + + + + + + + + + + + + + + + + + + {reports.map((r, i) => ( + + + + + + + + + + + + + + + + + ))} + +
user_nameemailprosthesis_modelprosthesis_serialinstallation_datetotal_usage_minutesusage_dateavg_battery_levelavg_max_forcelast_activetotal_error_counttotal_sessionsupdated_atcreated_at
{r.user_name}{r.email}{r.prosthesis_model}{r.prosthesis_serial}{r.installation_date}{r.total_usage_minutes}{r.usage_date}{r.avg_battery_level}{r.avg_max_force}{r.last_active}{r.total_error_count}{r.total_sessions}{r.updated_at}{r.created_at}
+ )} + +
+ +
+ +
); diff --git a/keycloak/realm-export.json b/keycloak/realm-export.json index 9646a4b17..7684de6e8 100644 --- a/keycloak/realm-export.json +++ b/keycloak/realm-export.json @@ -116,7 +116,31 @@ "publicClient": true, "redirectUris": ["http://localhost:3000/*"], "webOrigins": ["http://localhost:3000"], - "directAccessGrantsEnabled": true + "directAccessGrantsEnabled": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "attributes": { + "pkce.code.challenge.method": "S256", + "access.token.claim": "true" + }, + "defaultClientScopes": [ + "reports-api-audience" + ], + "protocolMappers": [ + { + "name": "realm roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "user.attribute": "role", + "access.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String" + } + } + ] }, { "clientId": "reports-api", @@ -125,5 +149,28 @@ "secret": "oNwoLQdvJAvRcL89SydqCWCe5ry1jMgq", "bearerOnly": true } + ], + "clientScopes": [ + { + "name": "reports-api-audience", + "description": "Adds audience 'reports-api'", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true" + }, + "protocolMappers": [ + { + "name": "audience-reports-api", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.client.audience": "reports-api", + "id.token.claim": "false", + "access.token.claim": "true" + } + } + ] + } ] } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 000000000..818c08a51 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "architecture-bionicpro", + "lockfileVersion": 3, + "requires": true, + "packages": {} +}