Initial

extract_logs.py

+import os
+import re
+import sqlite3
+from urllib.parse import parse_qs, urlencode
+
+
+def clean_query(query_string):
+    # Parse die Query-Parameter
+    params = parse_qs(query_string)
+
+    # Entferne unerwünschte Parameter
+    for param in ["shard", "shard.url", "isShard", "NOW", "wt"]:
+        params.pop(param, None)
+
+    # Konvertiere zurück zu einem Query-String
+    return urlencode(params, doseq=True)
+
+
+def extract_urls_from_logs():
+    log_dir = "logs/"
+    log_pattern = re.compile(r"solr\.log\.\d+")
+    urls = []
+
+    for filename in os.listdir(log_dir):
+        if log_pattern.match(filename):
+            with open(os.path.join(log_dir, filename), "r") as file:
+                for line in file:
+                    if "o.a.s.c.S.Request" in line and "params=" in line:
+                        params_match = re.search(r"params=\{(.+?)\}", line)
+                        if params_match:
+                            query_string = params_match.group(1)
+                            clean_query_string = clean_query(query_string)
+                            urls.append(f"/select?{clean_query_string}")
+
+    return urls
+
+
+def save_to_sqlite(urls):
+    conn = sqlite3.connect("solr_queries.db")
+    c = conn.cursor()
+
+    # Erstelle die Tabelle, falls sie nicht existiert
+    c.execute("""CREATE TABLE IF NOT EXISTS queries
+                 (id INTEGER PRIMARY KEY, url TEXT)""")
+
+    # Füge die URLs in die Datenbank ein
+    for url in urls:
+        c.execute("INSERT INTO queries (url) VALUES (?)", (url,))
+
+    conn.commit()
+    conn.close()
+
+
+if __name__ == "__main__":
+    urls = extract_urls_from_logs()
+    save_to_sqlite(urls)
+    print(f"{len(urls)} URLs wurden extrahiert und in die Datenbank gespeichert.")

locustfile.py

+import sqlite3
+from locust import HttpUser, task, between
+import random
+
+
+class SolrUser(HttpUser):
+    wait_time = between(1, 5)
+
+    def on_start(self):
+        self.queries = self.load_queries_from_db()
+
+    def load_queries_from_db(self):
+        conn = sqlite3.connect("solr_queries.db")
+        c = conn.cursor()
+        c.execute("SELECT url FROM queries")
+        queries = [row[0] for row in c.fetchall()]
+        conn.close()
+        return queries
+
+    @task
+    def search_solr(self):
+        if self.queries:
+            query = random.choice(self.queries)
+            self.client.get(query)
+        else:
+            print("Keine Abfragen in der Datenbank gefunden.")

requirements.txt

+locust
+locust-plugins