breakpilot-lehrer/edu-search-service/scripts/add_german_universities.py

#!/usr/bin/env python3
"""
Add all major German universities to the edu-search-service database.
Based on HRK (Hochschulrektorenkonferenz) list.
"""

import requests
import json
import time
import sys

API_BASE = "https://macmini:8089/api/v1"

# German Universities - categorized
GERMAN_UNIVERSITIES = [
    # === UNIVERSITIES (Universitäten) ===
    # Already in DB (skip or update)
    # {"name": "TUM", "url": "https://www.tum.de", "type": "university"},
    # {"name": "LMU München", "url": "https://www.lmu.de", "type": "university"},
    # {"name": "UOL", "url": "https://uol.de", "type": "university"},
    # {"name": "KIT Karlsruhe", "url": "https://www.kit.edu", "type": "university"},

    # TU9 Universities
    {"name": "TU Dresden", "url": "https://tu-dresden.de", "type": "university"},
    {"name": "TU Braunschweig", "url": "https://www.tu-braunschweig.de", "type": "university"},
    {"name": "TU Darmstadt", "url": "https://www.tu-darmstadt.de", "type": "university"},
    {"name": "Leibniz Universität Hannover", "url": "https://www.uni-hannover.de", "type": "university"},
    {"name": "Universität Stuttgart", "url": "https://www.uni-stuttgart.de", "type": "university"},

    # Excellence Universities
    {"name": "Universität Bonn", "url": "https://www.uni-bonn.de", "type": "university"},
    {"name": "Universität Konstanz", "url": "https://www.uni-konstanz.de", "type": "university"},
    {"name": "Universität Tübingen", "url": "https://uni-tuebingen.de", "type": "university"},
    {"name": "Universität Freiburg", "url": "https://www.uni-freiburg.de", "type": "university"},

    # Large State Universities
    {"name": "Universität Münster", "url": "https://www.uni-muenster.de", "type": "university"},
    {"name": "Universität Frankfurt", "url": "https://www.uni-frankfurt.de", "type": "university"},
    {"name": "Universität Mainz", "url": "https://www.uni-mainz.de", "type": "university"},
    {"name": "Universität Würzburg", "url": "https://www.uni-wuerzburg.de", "type": "university"},
    {"name": "Universität Erlangen-Nürnberg", "url": "https://www.fau.de", "type": "university"},
    {"name": "Universität Leipzig", "url": "https://www.uni-leipzig.de", "type": "university"},
    {"name": "Universität Jena", "url": "https://www.uni-jena.de", "type": "university"},
    {"name": "Universität Halle", "url": "https://www.uni-halle.de", "type": "university"},
    {"name": "Universität Rostock", "url": "https://www.uni-rostock.de", "type": "university"},
    {"name": "Universität Greifswald", "url": "https://www.uni-greifswald.de", "type": "university"},
    {"name": "Universität Kiel", "url": "https://www.uni-kiel.de", "type": "university"},
    {"name": "Universität Bremen", "url": "https://www.uni-bremen.de", "type": "university"},
    {"name": "Universität Bielefeld", "url": "https://www.uni-bielefeld.de", "type": "university"},
    {"name": "Universität Duisburg-Essen", "url": "https://www.uni-due.de", "type": "university"},
    {"name": "Universität Dortmund", "url": "https://www.tu-dortmund.de", "type": "university"},
    {"name": "Universität Bochum", "url": "https://www.ruhr-uni-bochum.de", "type": "university"},
    {"name": "Universität Düsseldorf", "url": "https://www.hhu.de", "type": "university"},
    {"name": "Universität Wuppertal", "url": "https://www.uni-wuppertal.de", "type": "university"},
    {"name": "Universität Siegen", "url": "https://www.uni-siegen.de", "type": "university"},
    {"name": "Universität Paderborn", "url": "https://www.uni-paderborn.de", "type": "university"},
    {"name": "Universität Kassel", "url": "https://www.uni-kassel.de", "type": "university"},
    {"name": "Universität Marburg", "url": "https://www.uni-marburg.de", "type": "university"},
    {"name": "Universität Gießen", "url": "https://www.uni-giessen.de", "type": "university"},
    {"name": "Universität Saarbrücken", "url": "https://www.uni-saarland.de", "type": "university"},
    {"name": "Universität Trier", "url": "https://www.uni-trier.de", "type": "university"},
    {"name": "Universität Koblenz", "url": "https://www.uni-koblenz.de", "type": "university"},
    {"name": "Universität Landau", "url": "https://rptu.de", "type": "university"},
    {"name": "Universität Mannheim", "url": "https://www.uni-mannheim.de", "type": "university"},
    {"name": "Universität Ulm", "url": "https://www.uni-ulm.de", "type": "university"},
    {"name": "Universität Hohenheim", "url": "https://www.uni-hohenheim.de", "type": "university"},
    {"name": "Universität Regensburg", "url": "https://www.uni-regensburg.de", "type": "university"},
    {"name": "Universität Passau", "url": "https://www.uni-passau.de", "type": "university"},
    {"name": "Universität Bayreuth", "url": "https://www.uni-bayreuth.de", "type": "university"},
    {"name": "Universität Bamberg", "url": "https://www.uni-bamberg.de", "type": "university"},
    {"name": "Universität Augsburg", "url": "https://www.uni-augsburg.de", "type": "university"},
    {"name": "Universität Potsdam", "url": "https://www.uni-potsdam.de", "type": "university"},
    {"name": "Universität Magdeburg", "url": "https://www.ovgu.de", "type": "university"},
    {"name": "TU Chemnitz", "url": "https://www.tu-chemnitz.de", "type": "university"},
    {"name": "TU Ilmenau", "url": "https://www.tu-ilmenau.de", "type": "university"},
    {"name": "TU Freiberg", "url": "https://tu-freiberg.de", "type": "university"},
    {"name": "TU Clausthal", "url": "https://www.tu-clausthal.de", "type": "university"},
    {"name": "TU Kaiserslautern", "url": "https://rptu.de", "type": "university"},
    {"name": "BTU Cottbus-Senftenberg", "url": "https://www.b-tu.de", "type": "university"},
    {"name": "Universität der Bundeswehr München", "url": "https://www.unibw.de", "type": "university"},
    {"name": "Universität der Bundeswehr Hamburg", "url": "https://www.hsu-hh.de", "type": "university"},

    # === FACHHOCHSCHULEN / HAW ===
    {"name": "HAW Hamburg", "url": "https://www.haw-hamburg.de", "type": "haw"},
    {"name": "HTW Berlin", "url": "https://www.htw-berlin.de", "type": "haw"},
    {"name": "Beuth Hochschule Berlin", "url": "https://www.bht-berlin.de", "type": "haw"},
    {"name": "FH Aachen", "url": "https://www.fh-aachen.de", "type": "haw"},
    {"name": "TH Köln", "url": "https://www.th-koeln.de", "type": "haw"},
    {"name": "Hochschule Düsseldorf", "url": "https://www.hs-duesseldorf.de", "type": "haw"},
    {"name": "FH Dortmund", "url": "https://www.fh-dortmund.de", "type": "haw"},
    {"name": "Hochschule Bochum", "url": "https://www.hochschule-bochum.de", "type": "haw"},
    {"name": "Westfälische Hochschule", "url": "https://www.w-hs.de", "type": "haw"},
    {"name": "FH Bielefeld", "url": "https://www.fh-bielefeld.de", "type": "haw"},
    {"name": "FH Münster", "url": "https://www.fh-muenster.de", "type": "haw"},
    {"name": "Hochschule Osnabrück", "url": "https://www.hs-osnabrueck.de", "type": "haw"},
    {"name": "Hochschule Bremen", "url": "https://www.hs-bremen.de", "type": "haw"},
    {"name": "Hochschule Hannover", "url": "https://www.hs-hannover.de", "type": "haw"},
    {"name": "Ostfalia Hochschule", "url": "https://www.ostfalia.de", "type": "haw"},
    {"name": "Hochschule Emden/Leer", "url": "https://www.hs-emden-leer.de", "type": "haw"},
    {"name": "HAWK Hildesheim", "url": "https://www.hawk.de", "type": "haw"},
    {"name": "Hochschule Fulda", "url": "https://www.hs-fulda.de", "type": "haw"},
    {"name": "Frankfurt UAS", "url": "https://www.frankfurt-university.de", "type": "haw"},
    {"name": "Hochschule Darmstadt", "url": "https://www.h-da.de", "type": "haw"},
    {"name": "Hochschule RheinMain", "url": "https://www.hs-rm.de", "type": "haw"},
    {"name": "Hochschule Mainz", "url": "https://www.hs-mainz.de", "type": "haw"},
    {"name": "Hochschule Trier", "url": "https://www.hochschule-trier.de", "type": "haw"},
    {"name": "Hochschule Koblenz", "url": "https://www.hs-koblenz.de", "type": "haw"},
    {"name": "Hochschule Karlsruhe", "url": "https://www.h-ka.de", "type": "haw"},
    {"name": "Hochschule Mannheim", "url": "https://www.hs-mannheim.de", "type": "haw"},
    {"name": "Hochschule Heilbronn", "url": "https://www.hs-heilbronn.de", "type": "haw"},
    {"name": "Hochschule Esslingen", "url": "https://www.hs-esslingen.de", "type": "haw"},
    {"name": "Hochschule Reutlingen", "url": "https://www.reutlingen-university.de", "type": "haw"},
    {"name": "Hochschule Konstanz", "url": "https://www.htwg-konstanz.de", "type": "haw"},
    {"name": "Hochschule Offenburg", "url": "https://www.hs-offenburg.de", "type": "haw"},
    {"name": "Hochschule Pforzheim", "url": "https://www.hs-pforzheim.de", "type": "haw"},
    {"name": "Hochschule Albstadt-Sigmaringen", "url": "https://www.hs-albsig.de", "type": "haw"},
    {"name": "Hochschule München", "url": "https://www.hm.edu", "type": "haw"},
    {"name": "TH Nürnberg", "url": "https://www.th-nuernberg.de", "type": "haw"},
    {"name": "TH Ingolstadt", "url": "https://www.thi.de", "type": "haw"},
    {"name": "Hochschule Augsburg", "url": "https://www.hs-augsburg.de", "type": "haw"},
    {"name": "Hochschule Rosenheim", "url": "https://www.th-rosenheim.de", "type": "haw"},
    {"name": "Hochschule Regensburg", "url": "https://www.oth-regensburg.de", "type": "haw"},
    {"name": "Hochschule Landshut", "url": "https://www.haw-landshut.de", "type": "haw"},
    {"name": "Hochschule Coburg", "url": "https://www.hs-coburg.de", "type": "haw"},
    {"name": "Hochschule Hof", "url": "https://www.hof-university.de", "type": "haw"},
    {"name": "Hochschule Würzburg-Schweinfurt", "url": "https://www.thws.de", "type": "haw"},
    {"name": "Hochschule Aschaffenburg", "url": "https://www.th-ab.de", "type": "haw"},
    {"name": "Hochschule Ansbach", "url": "https://www.hs-ansbach.de", "type": "haw"},
    {"name": "OTH Amberg-Weiden", "url": "https://www.oth-aw.de", "type": "haw"},
    {"name": "Hochschule Deggendorf", "url": "https://www.th-deg.de", "type": "haw"},
    {"name": "Hochschule Kempten", "url": "https://www.hs-kempten.de", "type": "haw"},
    {"name": "Hochschule Neu-Ulm", "url": "https://www.hnu.de", "type": "haw"},
    {"name": "HTW Dresden", "url": "https://www.htw-dresden.de", "type": "haw"},
    {"name": "HTWK Leipzig", "url": "https://www.htwk-leipzig.de", "type": "haw"},
    {"name": "Hochschule Mittweida", "url": "https://www.hs-mittweida.de", "type": "haw"},
    {"name": "Hochschule Zittau/Görlitz", "url": "https://www.hszg.de", "type": "haw"},
    {"name": "Westsächsische Hochschule Zwickau", "url": "https://www.fh-zwickau.de", "type": "haw"},
    {"name": "Hochschule Merseburg", "url": "https://www.hs-merseburg.de", "type": "haw"},
    {"name": "Hochschule Anhalt", "url": "https://www.hs-anhalt.de", "type": "haw"},
    {"name": "Hochschule Magdeburg-Stendal", "url": "https://www.h2.de", "type": "haw"},
    {"name": "Hochschule Harz", "url": "https://www.hs-harz.de", "type": "haw"},
    {"name": "Ernst-Abbe-Hochschule Jena", "url": "https://www.eah-jena.de", "type": "haw"},
    {"name": "FH Erfurt", "url": "https://www.fh-erfurt.de", "type": "haw"},
    {"name": "Hochschule Nordhausen", "url": "https://www.hs-nordhausen.de", "type": "haw"},
    {"name": "Hochschule Schmalkalden", "url": "https://www.hs-schmalkalden.de", "type": "haw"},
    {"name": "TH Brandenburg", "url": "https://www.th-brandenburg.de", "type": "haw"},
    {"name": "FH Potsdam", "url": "https://www.fh-potsdam.de", "type": "haw"},
    {"name": "TH Wildau", "url": "https://www.th-wildau.de", "type": "haw"},
    {"name": "Hochschule Neubrandenburg", "url": "https://www.hs-nb.de", "type": "haw"},
    {"name": "Hochschule Stralsund", "url": "https://www.hochschule-stralsund.de", "type": "haw"},
    {"name": "Hochschule Wismar", "url": "https://www.hs-wismar.de", "type": "haw"},
    {"name": "FH Kiel", "url": "https://www.fh-kiel.de", "type": "haw"},
    {"name": "FH Westküste", "url": "https://www.fh-westkueste.de", "type": "haw"},
    {"name": "TH Lübeck", "url": "https://www.th-luebeck.de", "type": "haw"},
    {"name": "FH Flensburg", "url": "https://hs-flensburg.de", "type": "haw"},
    {"name": "Hochschule Bremerhaven", "url": "https://www.hs-bremerhaven.de", "type": "haw"},

    # === PRIVATE HOCHSCHULEN ===
    {"name": "WHU Vallendar", "url": "https://www.whu.edu", "type": "private"},
    {"name": "HHL Leipzig", "url": "https://www.hhl.de", "type": "private"},
    {"name": "EBS Universität", "url": "https://www.ebs.edu", "type": "private"},
    {"name": "Frankfurt School", "url": "https://www.frankfurt-school.de", "type": "private"},
    {"name": "ESMT Berlin", "url": "https://esmt.berlin", "type": "private"},
    {"name": "Jacobs University Bremen", "url": "https://www.jacobs-university.de", "type": "private"},
    {"name": "Zeppelin Universität", "url": "https://www.zu.de", "type": "private"},
    {"name": "Bucerius Law School", "url": "https://www.law-school.de", "type": "private"},
    {"name": "Universität Witten/Herdecke", "url": "https://www.uni-wh.de", "type": "private"},
    {"name": "IUBH", "url": "https://www.iu.de", "type": "private"},
    {"name": "SRH Hochschule Heidelberg", "url": "https://www.srh-hochschule-heidelberg.de", "type": "private"},
    {"name": "FOM Hochschule", "url": "https://www.fom.de", "type": "private"},

    # === FRAUNHOFER INSTITUTE ===
    {"name": "Fraunhofer IIS", "url": "https://www.iis.fraunhofer.de", "type": "research"},
    {"name": "Fraunhofer IAIS", "url": "https://www.iais.fraunhofer.de", "type": "research"},
    {"name": "Fraunhofer IML", "url": "https://www.iml.fraunhofer.de", "type": "research"},
    {"name": "Fraunhofer ISI", "url": "https://www.isi.fraunhofer.de", "type": "research"},
    {"name": "Fraunhofer IPA", "url": "https://www.ipa.fraunhofer.de", "type": "research"},
    {"name": "Fraunhofer IAO", "url": "https://www.iao.fraunhofer.de", "type": "research"},
    {"name": "Fraunhofer IWS", "url": "https://www.iws.fraunhofer.de", "type": "research"},
    {"name": "Fraunhofer IPT", "url": "https://www.ipt.fraunhofer.de", "type": "research"},
    {"name": "Fraunhofer FOKUS", "url": "https://www.fokus.fraunhofer.de", "type": "research"},
    {"name": "Fraunhofer HHI", "url": "https://www.hhi.fraunhofer.de", "type": "research"},
    {"name": "Fraunhofer IESE", "url": "https://www.iese.fraunhofer.de", "type": "research"},
    {"name": "Fraunhofer IOSB", "url": "https://www.iosb.fraunhofer.de", "type": "research"},
    {"name": "Fraunhofer IDMT", "url": "https://www.idmt.fraunhofer.de", "type": "research"},
    {"name": "Fraunhofer IKTS", "url": "https://www.ikts.fraunhofer.de", "type": "research"},
    {"name": "Fraunhofer IGD", "url": "https://www.igd.fraunhofer.de", "type": "research"},

    # === MAX-PLANCK-INSTITUTE ===
    {"name": "MPI für Informatik", "url": "https://www.mpi-inf.mpg.de", "type": "research"},
    {"name": "MPI für Software Systeme", "url": "https://www.mpi-sws.org", "type": "research"},
    {"name": "MPI für intelligente Systeme", "url": "https://is.mpg.de", "type": "research"},
    {"name": "MPI für Mathematik", "url": "https://www.mpim-bonn.mpg.de", "type": "research"},
    {"name": "MPI für Physik", "url": "https://www.mpp.mpg.de", "type": "research"},
    {"name": "MPI für Quantenoptik", "url": "https://www.mpq.mpg.de", "type": "research"},
    {"name": "MPI für Biophysik", "url": "https://www.biophys.mpg.de", "type": "research"},
    {"name": "MPI für Biochemie", "url": "https://www.biochem.mpg.de", "type": "research"},
    {"name": "MPI für Neurobiologie", "url": "https://www.neuro.mpg.de", "type": "research"},
    {"name": "MPI für Hirnforschung", "url": "https://brain.mpg.de", "type": "research"},

    # === HELMHOLTZ-ZENTREN ===
    {"name": "DESY Hamburg", "url": "https://www.desy.de", "type": "research"},
    {"name": "FZ Jülich", "url": "https://www.fz-juelich.de", "type": "research"},
    {"name": "GSI Darmstadt", "url": "https://www.gsi.de", "type": "research"},
    {"name": "DKFZ Heidelberg", "url": "https://www.dkfz.de", "type": "research"},
    {"name": "DLR", "url": "https://www.dlr.de", "type": "research"},
    {"name": "AWI Bremerhaven", "url": "https://www.awi.de", "type": "research"},
    {"name": "GFZ Potsdam", "url": "https://www.gfz-potsdam.de", "type": "research"},
    {"name": "UFZ Leipzig", "url": "https://www.ufz.de", "type": "research"},
    {"name": "GEOMAR Kiel", "url": "https://www.geomar.de", "type": "research"},
]

def get_existing_universities():
    """Get list of existing universities from the API."""
    try:
        response = requests.get(f"{API_BASE}/universities", verify=False, timeout=10)
        if response.status_code == 200:
            data = response.json()
            return {u['url'].rstrip('/').lower(): u for u in data.get('universities', [])}
    except Exception as e:
        print(f"Error fetching existing universities: {e}")
    return {}

def add_university(uni):
    """Add a university to the database."""
    payload = {
        "name": uni["name"],
        "url": uni["url"],
        "type": uni.get("type", "university"),
        "country": "DE"
    }

    try:
        response = requests.post(
            f"{API_BASE}/universities",
            json=payload,
            verify=False,
            timeout=10
        )
        return response.status_code == 201 or response.status_code == 200
    except Exception as e:
        print(f"Error adding {uni['name']}: {e}")
        return False

def main():
    print("Fetching existing universities...")
    existing = get_existing_universities()
    print(f"Found {len(existing)} existing universities")

    added = 0
    skipped = 0
    failed = 0

    for uni in GERMAN_UNIVERSITIES:
        url_key = uni["url"].rstrip('/').lower()

        if url_key in existing:
            print(f"SKIP: {uni['name']} (already exists)")
            skipped += 1
            continue

        print(f"ADD:  {uni['name']} ({uni['url']})")
        if add_university(uni):
            added += 1
        else:
            failed += 1

        # Rate limiting
        time.sleep(0.2)

    print(f"\n=== SUMMARY ===")
    print(f"Added:   {added}")
    print(f"Skipped: {skipped}")
    print(f"Failed:  {failed}")
    print(f"Total:   {len(GERMAN_UNIVERSITIES)}")

if __name__ == "__main__":
    # Disable SSL warnings for self-signed cert
    import urllib3
    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

    main()