Integrate FR24 data

devloop/parse_retrieve.py

@@ -1,12 +1,15 @@
 # -*- encoding: utf-8 -*-
 import os
 
-from latecomers.retrieve import retrieve
+from latecomers.retrieve import retrieve_from_inst, retrieve_from_fr24
-from latecomers.parse import find_table, get_details
+from latecomers.parse import find_table, get_details, parse_fr24
 
-body = retrieve()
+body = retrieve_from_inst()
 
 flights = find_table(body)
 
+aux_data = parse_fr24(retrieve_from_fr24())
+
+breakpoint()
 for f in flights:
-    print(get_details(f, os.environ.get("DEBUG") is not None))
+    print(get_details(f, aux_data=aux_data, debug=os.environ.get("DEBUG") is not None))

devloop/retrieve.py

@@ -1,6 +1,7 @@
 # -*- encoding: utf-8 -*-
-from latecomers.retrieve import retrieve
+from latecomers.retrieve import retrieve_from_inst, retrieve_from_fr24
 
 
-body = retrieve()
+body = retrieve_from_inst()
+fr24_data = retrieve_from_fr24()
 print(body)

latecomers/main.py

@@ -3,8 +3,8 @@ import logging
 import sys
 import typing as T
 
-from latecomers.retrieve import retrieve
+from latecomers.retrieve import retrieve_from_inst, retrieve_from_fr24
-from latecomers.parse import find_table, get_details, Details
+from latecomers.parse import find_table, get_details, Details, parse_fr24
 from latecomers.serializer import to_excel
 from latecomers.notifier import Notifier
 from latecomers.config import Config
@@ -23,11 +23,13 @@ def main(config: Config):
     The main cli entrypoint.
     """
     out = Notifier(**config.smtp)
-    body = retrieve()
+    body = retrieve_from_inst()
     table = find_table(body)
+    fr24_data = retrieve_from_fr24()
+    aux_data = parse_fr24(fr24_data)
     data: T.List[Details] = []
     for row in table:
-        data.append(get_details(row))
+        data.append(get_details(row, aux_data))
 
     if not data:
         out.send_no_data(config.to)

latecomers/parse.py

@@ -1,5 +1,6 @@
 # -*- encoding: utf-8 -*-
 from dataclasses import dataclass
+from datetime import datetime
 from enum import Enum
 import logging
 import re
@@ -71,6 +72,7 @@ class Details(object):
     code: T.Optional[T.Text] = None
     origin: T.Optional[T.Text] = None
     status: Status = Status.UNKNOWN
+    fr24_landing_time: T.Optional[T.Text] = None
 
     def maybe_parse_hour_th(self, h5: et._ElementTree) -> None:
         """
@@ -101,7 +103,7 @@ class Details(object):
             return
         child = h5.xpath(".//strong")
         if len(child) == 1:
-            self.code = child[0].text.strip("\t\n ")
+            self.code = child[0].text.strip("\t\n ").replace(" ", "")
 
     def maybe_parse_airport(self, h5: et._ElementTree) -> None:
         """
@@ -125,20 +127,37 @@ class Details(object):
             if len(parsed) == 1:
                 self.status = Status.from_str(parsed[0])
 
+    def maybe_add_aux_data(self, aux_data: T.Dict[T.Text, T.Text]) -> None:
+        """
+        This function extends the current data with auxiliary sources (currently
+        only FlightRadar24 data).
+        """
+        if not self.code:
+            return
+
+        self.fr24_landing_time = aux_data.get(self.code)
+
     def __str__(self) -> T.Text:
         res: T.Dict[T.Text, T.Optional[T.Text]] = {}
         if self.th_arrival:
             res["theoric"] = self.th_arrival
-        res["real"] = self.real_arrival
+        if self.real_arrival:
+            res["real"] = self.real_arrival
         if self.code:
             res["code"] = self.code
         res["origin"] = self.origin
         res["status"] = self.status.value
+        if self.fr24_landing_time:
+            res["fr24_landing_time"] = self.fr24_landing_time
         desc = ",".join([f"{k}={v}" for k, v in res.items()])
         return f"Detail<{desc}>"
 
 
-def get_details(table_entry: et._ElementTree, debug: bool = False) -> Details:
+def get_details(
+    table_entry: et._ElementTree,
+    aux_data: T.Optional[T.Dict[T.Text, T.Text]] = None,
+    debug: bool = False,
+) -> Details:
     """
     Find the dates in a table row. If a strikenthrough time is found, it is
     returned as second element in the tuple.
@@ -166,4 +185,44 @@ def get_details(table_entry: et._ElementTree, debug: bool = False) -> Details:
         d.maybe_parse_airport(res[3])
         d.maybe_parse_status(res[4])
 
+    if aux_data:
+        d.maybe_add_aux_data(aux_data)
+
     return d
+
+
+def parse_fr24(
+    data: T.Optional[T.Dict[T.Text, T.Any]]
+) -> T.Optional[T.Dict[T.Text, T.Text]]:
+    """
+    This function parses the given FlightRadar24 data into a pandas DataFrame.
+    """
+    if not data:
+        return None
+
+    try:
+        results = {}
+        for flight in data["result"]["response"]["airport"]["pluginData"]["schedule"][
+            "arrivals"
+        ][
+            "data"
+        ]:  # noqa: E501
+            try:
+                id_num = flight["flight"]["identification"]["number"]
+                if (_code := id_num.get("default")):
+                    code = _code
+                elif (_code := id_num.get("alternative")):
+                    code = _code
+                else:
+                    # skip if no flight code found
+                    continue
+                ts = flight["flight"]["time"]["real"]["arrival"]
+                real_arrival = datetime.fromtimestamp(ts).strftime("%H:%M")
+                results[code] = real_arrival
+            except:  # noqa: E722
+                continue
+
+        return results
+
+    except:  # noqa: E722
+        return None

latecomers/serializer.py

@@ -25,6 +25,7 @@ def to_excel(data: T.List[Details]) -> bytes:
         "code": "Codice volo",
         "origin": "Aeroporto di partenza",
         "status": "Stato",
+        "fr24_landing_time": "Ora atterraggio (FlightRadar24)",
     }
     df = pd.DataFrame(data, columns=mapping)
     df["status"] = df["status"].map(lambda x: x.value)