Balken

extract_logs.py

@@ -5,6 +5,7 @@ import pandas as pd
 import concurrent.futures
 import pyarrow as pa
 import pyarrow.parquet as pq
+from tqdm import tqdm
 
 
 def clean_query(query_string):
@@ -73,18 +74,23 @@ def extract_urls_from_logs():
 
     total_urls = 0
     with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
-        future_to_file = {
+        futures = {
             executor.submit(process_file, file): file for file in files_to_process
         }
-        for future in concurrent.futures.as_completed(future_to_file):
-            file = future_to_file[future]
+
+        # Erstelle einen Fortschrittsbalken für die Gesamtanzahl der Dateien
+        with tqdm(total=len(files_to_process), desc="Gesamtfortschritt") as pbar:
+            for future in concurrent.futures.as_completed(futures):
+                file = futures[future]
                 try:
                     df = future.result()
                     total_urls += len(df)
                     table = pa.Table.from_pandas(df)
                     writer.write_table(table)
+                    pbar.update(1)
+                    pbar.set_postfix({"Datei": os.path.basename(file), "URLs": len(df)})
                 except Exception as exc:
-                print(f"{file} generated an exception: {exc}")
+                    print(f"{file} generierte eine Ausnahme: {exc}")
 
     writer.close()
     return total_urls

requirements.txt

@@ -2,3 +2,4 @@ locust
 locust-plugins
 pandas
 pyarrow
+tqdm
\ No newline at end of file