Integrate FR24 data

master
blallo 2022-09-07 23:44:53 +02:00
parent e916f8628b
commit 432396c836
Signed by: blallo
GPG Key ID: 0CBE577C9B72DC3F
6 changed files with 99 additions and 17 deletions

View File

@ -1,12 +1,15 @@
# -*- encoding: utf-8 -*-
import os
from latecomers.retrieve import retrieve
from latecomers.parse import find_table, get_details
from latecomers.retrieve import retrieve_from_inst, retrieve_from_fr24
from latecomers.parse import find_table, get_details, parse_fr24
body = retrieve()
body = retrieve_from_inst()
flights = find_table(body)
aux_data = parse_fr24(retrieve_from_fr24())
breakpoint()
for f in flights:
print(get_details(f, os.environ.get("DEBUG") is not None))
print(get_details(f, aux_data=aux_data, debug=os.environ.get("DEBUG") is not None))

View File

@ -1,6 +1,7 @@
# -*- encoding: utf-8 -*-
from latecomers.retrieve import retrieve
from latecomers.retrieve import retrieve_from_inst, retrieve_from_fr24
body = retrieve()
body = retrieve_from_inst()
fr24_data = retrieve_from_fr24()
print(body)

View File

@ -3,8 +3,8 @@ import logging
import sys
import typing as T
from latecomers.retrieve import retrieve
from latecomers.parse import find_table, get_details, Details
from latecomers.retrieve import retrieve_from_inst, retrieve_from_fr24
from latecomers.parse import find_table, get_details, Details, parse_fr24
from latecomers.serializer import to_excel
from latecomers.notifier import Notifier
from latecomers.config import Config
@ -23,11 +23,13 @@ def main(config: Config):
The main cli entrypoint.
"""
out = Notifier(**config.smtp)
body = retrieve()
body = retrieve_from_inst()
table = find_table(body)
fr24_data = retrieve_from_fr24()
aux_data = parse_fr24(fr24_data)
data: T.List[Details] = []
for row in table:
data.append(get_details(row))
data.append(get_details(row, aux_data))
if not data:
out.send_no_data(config.to)

View File

@ -1,5 +1,6 @@
# -*- encoding: utf-8 -*-
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
import logging
import re
@ -71,6 +72,7 @@ class Details(object):
code: T.Optional[T.Text] = None
origin: T.Optional[T.Text] = None
status: Status = Status.UNKNOWN
fr24_landing_time: T.Optional[T.Text] = None
def maybe_parse_hour_th(self, h5: et._ElementTree) -> None:
"""
@ -101,7 +103,7 @@ class Details(object):
return
child = h5.xpath(".//strong")
if len(child) == 1:
self.code = child[0].text.strip("\t\n ")
self.code = child[0].text.strip("\t\n ").replace(" ", "")
def maybe_parse_airport(self, h5: et._ElementTree) -> None:
"""
@ -125,20 +127,37 @@ class Details(object):
if len(parsed) == 1:
self.status = Status.from_str(parsed[0])
def maybe_add_aux_data(self, aux_data: T.Dict[T.Text, T.Text]) -> None:
"""
This function extends the current data with auxiliary sources (currently
only FlightRadar24 data).
"""
if not self.code:
return
self.fr24_landing_time = aux_data.get(self.code)
def __str__(self) -> T.Text:
res: T.Dict[T.Text, T.Optional[T.Text]] = {}
if self.th_arrival:
res["theoric"] = self.th_arrival
if self.real_arrival:
res["real"] = self.real_arrival
if self.code:
res["code"] = self.code
res["origin"] = self.origin
res["status"] = self.status.value
if self.fr24_landing_time:
res["fr24_landing_time"] = self.fr24_landing_time
desc = ",".join([f"{k}={v}" for k, v in res.items()])
return f"Detail<{desc}>"
def get_details(table_entry: et._ElementTree, debug: bool = False) -> Details:
def get_details(
table_entry: et._ElementTree,
aux_data: T.Optional[T.Dict[T.Text, T.Text]] = None,
debug: bool = False,
) -> Details:
"""
Find the dates in a table row. If a strikenthrough time is found, it is
returned as second element in the tuple.
@ -166,4 +185,44 @@ def get_details(table_entry: et._ElementTree, debug: bool = False) -> Details:
d.maybe_parse_airport(res[3])
d.maybe_parse_status(res[4])
if aux_data:
d.maybe_add_aux_data(aux_data)
return d
def parse_fr24(
data: T.Optional[T.Dict[T.Text, T.Any]]
) -> T.Optional[T.Dict[T.Text, T.Text]]:
"""
This function parses the given FlightRadar24 data into a pandas DataFrame.
"""
if not data:
return None
try:
results = {}
for flight in data["result"]["response"]["airport"]["pluginData"]["schedule"][
"arrivals"
][
"data"
]: # noqa: E501
try:
id_num = flight["flight"]["identification"]["number"]
if (_code := id_num.get("default")):
code = _code
elif (_code := id_num.get("alternative")):
code = _code
else:
# skip if no flight code found
continue
ts = flight["flight"]["time"]["real"]["arrival"]
real_arrival = datetime.fromtimestamp(ts).strftime("%H:%M")
results[code] = real_arrival
except: # noqa: E722
continue
return results
except: # noqa: E722
return None

File diff suppressed because one or more lines are too long

View File

@ -25,6 +25,7 @@ def to_excel(data: T.List[Details]) -> bytes:
"code": "Codice volo",
"origin": "Aeroporto di partenza",
"status": "Stato",
"fr24_landing_time": "Ora atterraggio (FlightRadar24)",
}
df = pd.DataFrame(data, columns=mapping)
df["status"] = df["status"].map(lambda x: x.value)