From 534ab1ee2b14a6d3ea204181588c5c3b34349d03 Mon Sep 17 00:00:00 2001 From: valexande Date: Thu, 10 Oct 2024 23:00:26 +0000 Subject: [PATCH] Upload files to "meteorogical_data" --- meteorogical_data/device_data_rdf.py | 97 +++++++++++ meteorogical_data/devices.json | 248 +++++++++++++++++++++++++++ meteorogical_data/helper_meteo.py | 73 ++++++++ 3 files changed, 418 insertions(+) create mode 100644 meteorogical_data/device_data_rdf.py create mode 100644 meteorogical_data/devices.json create mode 100644 meteorogical_data/helper_meteo.py diff --git a/meteorogical_data/device_data_rdf.py b/meteorogical_data/device_data_rdf.py new file mode 100644 index 0000000..e89c351 --- /dev/null +++ b/meteorogical_data/device_data_rdf.py @@ -0,0 +1,97 @@ +from rdflib import Graph, Namespace, URIRef, Literal, RDF, RDFS, XSD +from SPARQLWrapper import SPARQLWrapper, POST, BASIC, URLENCODED + +class device(): + + + def __init__(self, device_data_json, database, device_id): + self.device_data_json = device_data_json + self.database = database + self.device_id = device_id + + def read_ttl_and_store_in_graphdb(self, ttl_file_path, graphdb_endpoint, graphdb_repository, username=None, + password=None): + """ + Reads RDF data from a Turtle file and stores it in a GraphDB repository. + + :param ttl_file_path: Path to the Turtle file. + :param graphdb_endpoint: URL of the GraphDB endpoint. + :param graphdb_repository: Name of the GraphDB repository. + :param username: (Optional) Username for GraphDB authentication. + :param password: (Optional) Password for GraphDB authentication. + """ + # Create a graph + graph = Graph() + + # Parse the Turtle file + graph.parse(ttl_file_path, format="turtle") + + # Serialize the graph to a string in N-Triples format + ntriples_data = graph.serialize(format='nt') + + # If ntriples_data is bytes, decode it to string + if isinstance(ntriples_data, bytes): + ntriples_data = ntriples_data.decode('utf-8') + + # Create a SPARQLWrapper instance for the GraphDB endpoint + sparql = SPARQLWrapper(f"{graphdb_endpoint}/repositories/{graphdb_repository}/statements") + sparql.setMethod(POST) + sparql.setRequestMethod(URLENCODED) + sparql.addParameter('update', f""" + INSERT DATA {{ + {ntriples_data} + }} + """) + + # Set credentials if provided + if username and password: + sparql.setHTTPAuth(BASIC) + sparql.setCredentials(username, password) + + # Send the request to the GraphDB repository + sparql.query() + + def parse_device_data(self): + # Create a new RDF graph + g = Graph() + # Define a custom namespace + ontology = Namespace("http://sincere.org/s1#") + resource = Namespace("http://sincere.org/s1/resource/") + g.bind('sincere', ontology) + + g.add((resource["device_" + self.device_id], RDF.type, ontology.Device)) + g.add((resource["device_" + self.device_id], ontology.database, Literal(self.database, datatype=XSD.string))) + g.add((resource["device_" + self.device_id], RDFS.label, Literal(self.device_id, datatype=XSD.string))) + + for dev in self.device_data_json: + channel = dev["channel"] + channel_id = dev["channel_id"] + constant = dev["constant"] + device_id = dev["device_id"] + item = dev["item"] + unit = dev["unit"] + writable = dev["writable"] + g.add((resource["device_" + self.device_id], ontology.hasChannel, resource["channel_" + channel_id])) + g.add((resource["channel_" + channel_id], RDF.type, ontology.Channel)) + g.add((resource["channel_" + channel_id], ontology.number, Literal(channel, datatype=XSD.integer))) + g.add((resource["channel_" + channel_id], ontology.number, Literal(channel, datatype=XSD.integer))) + g.add((resource["channel_" + channel_id], ontology.constant, Literal(constant, datatype=XSD.string))) + g.add((resource["channel_" + channel_id], RDFS.label, Literal(channel_id, datatype=XSD.string))) + g.add((resource["channel_" + channel_id], ontology.device_id, Literal(device_id, datatype=XSD.string))) + g.add((resource["channel_" + channel_id], ontology.item, Literal(item, datatype=XSD.string))) + g.add((resource["channel_" + channel_id], ontology.unit, Literal(unit, datatype=XSD.string))) + g.add((resource["channel_" + channel_id], ontology.writable, Literal(writable, datatype=XSD.boolean))) + + # Create RDF triples + # Serialize and save the RDF graph to a TTL file + file_path = "output_device.ttl" + g.serialize(destination=file_path, format="turtle") + + graphdb_endpoint = 'http://localhost:7200' # Replace with the URL of your GraphDB endpoint + graphdb_repository = 'weather_measurement' # Replace with the name of your GraphDB repository + username = 'admin' # Replace with your GraphDB username (if required) + password = 'root' # Replace with your GraphDB password (if required) + + self.read_ttl_and_store_in_graphdb(file_path, graphdb_endpoint, graphdb_repository, username, password) + print("Data was mapped in GraphDB") + return 1 \ No newline at end of file diff --git a/meteorogical_data/devices.json b/meteorogical_data/devices.json new file mode 100644 index 0000000..2befa1d --- /dev/null +++ b/meteorogical_data/devices.json @@ -0,0 +1,248 @@ +{ + "status": 200, + "data": { + "reference": "DAVIS_Meteo", + "timer_loop": 120, + "measurement": "algete_meteo", + "file": "devis", + "gateway": "10.130.20.182", + "available": null, + "last_access": "2023-11-29T09:43:58.464780Z", + "channels": [ + { + "channel": 48, + "item": "ForecastIcons", + "unit": "None", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "0a0332f1-8a21-49fc-99e9-cf2807a1b4ce" + }, + { + "channel": 49, + "item": "ForecastRuleNumber", + "unit": "None", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "809e5a8a-13f9-418d-8578-a66e008602be" + }, + { + "channel": 50, + "item": "TimeSunrise", + "unit": "HHMM", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "47c3fce8-ca58-4849-9d43-99164fd9cd0f" + }, + { + "channel": 51, + "item": "TimeSunset", + "unit": "HHMM", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "a8c1479c-990f-4b9c-a0f0-bfd708ae010f" + }, + { + "channel": 52, + "item": "WetBulb", + "unit": "HHMM", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "8c3a9ef7-4913-4630-bd3c-fbf3e0188592" + }, + { + "channel": 62, + "item": "DewPoint", + "unit": "°F", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "fccfc8f9-b58a-495c-8cf3-817f5c6c439c" + }, + { + "channel": 63, + "item": "HeatIndex", + "unit": "°F", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "a894273e-d703-4ed0-bd50-e80617d19e77" + }, + { + "channel": 64, + "item": "WindChill", + "unit": "°F", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "a4f8ede7-7579-4dce-a812-bfa837d1ca1b" + }, + { + "channel": 65, + "item": "THSWIndex", + "unit": "°F", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "3dc3baff-520c-4f50-a222-cffeb4794dda" + }, + { + "channel": 66, + "item": "Last15minRain", + "unit": "Inches", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "62666981-2853-4117-9932-fa8104a4ab91" + }, + { + "channel": 67, + "item": "LastHourRain", + "unit": "Inches", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "3399e58e-9857-466c-969c-14a0717b56c4" + }, + { + "channel": 68, + "item": "Last24HourRain", + "unit": "Inches", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "7b1caebe-b42c-49f7-854e-2fac8f010603" + }, + { + "channel": 72, + "item": "BarometricSensorRawReading", + "unit": "Inches", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "c99843be-aa3d-4f0a-886f-dfbe54f90877" + }, + { + "channel": 73, + "item": "AbsoluteBarometricPressure", + "unit": "Inches", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "eaec7c4c-e9ba-4865-8b65-9bc57077dead" + }, + { + "channel": 2, + "item": "Barometer", + "unit": "inHg", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "4874b71d-2084-4967-a556-84c8772b5d18" + }, + { + "channel": 3, + "item": "InsideTemperature", + "unit": "Cº", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "3450f1e5-f7ab-45a4-a0f6-4c43cc6fceaa" + }, + { + "channel": 4, + "item": "InsideHumidity", + "unit": "%", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "bc9d8558-3268-484b-b96e-b161a587777d" + }, + { + "channel": 5, + "item": "OutsideTemperature", + "unit": "None", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "f6c8f25a-ef51-4e5a-a99e-f4c544675e1d" + }, + { + "channel": 6, + "item": "WindSpeed", + "unit": "None", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "3bf6013d-04f6-4601-8173-9ead59ed08ab" + }, + { + "channel": 7, + "item": "10MinAvrgWindSpeed", + "unit": "None", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "178c0dba-25d6-41ac-93db-6b43520b5756" + }, + { + "channel": 8, + "item": "WindDirection", + "unit": "None", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "64695b1a-960d-4ae9-a47b-3e5575070d20" + }, + { + "channel": 17, + "item": "OutsideHumidity", + "unit": "None", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "17faa574-9935-4db1-a239-8efbb2653b3e" + }, + { + "channel": 22, + "item": "RainRate", + "unit": "None", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "d267c1a4-efbc-4cd1-bb7b-98c6dc70d6e8" + }, + { + "channel": 23, + "item": "UVIndex", + "unit": "None", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "eb2f0322-a921-4d73-98a4-b6bc66a8dceb" + }, + { + "channel": 24, + "item": "SolarRadiation", + "unit": "W/m2", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "8a4c6b58-07e1-4cd5-9556-2fd7e75e6449" + }, + { + "channel": 25, + "item": "StormRain", + "unit": "Inches", + "constant": null, + "writable": false, + "device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e", + "channel_id": "db36ca06-d5a9-4bab-821a-4db134c212c1" + } + ] + } +} \ No newline at end of file diff --git a/meteorogical_data/helper_meteo.py b/meteorogical_data/helper_meteo.py new file mode 100644 index 0000000..03c0032 --- /dev/null +++ b/meteorogical_data/helper_meteo.py @@ -0,0 +1,73 @@ +import json +from rdflib import Graph, Literal, Namespace, RDF, URIRef, XSD +import os + +# Define the namespaces +SINCERE = Namespace("http://sincere.org/s1#") + + +# Function to convert JSON to RDF +def json_to_rdf(json_data, rdf_file_path): + # Create an RDF graph + g = Graph() + + # Bind the SINCERE namespace + g.bind("sincere", SINCERE) + + # Process each entry in the data + for i, entry in enumerate(json_data['data'], start=1): + channel_id = entry['channel_id'] + data_value = entry['data_value'] + measurement = entry['measurement'] + time = entry['time'] + + channel_uri = URIRef(f"http://sincere.org/s1/resource/channel_{channel_id}") + measurement_uri = URIRef(f"http://sincere.org/s1/resource/measurement_{channel_id}_{i}") + + g.add((channel_uri, RDF.type, SINCERE.Channel)) + g.add((channel_uri, SINCERE.hasMeasurement, measurement_uri)) + + g.add((measurement_uri, RDF.type, SINCERE.Measurement)) + g.add((measurement_uri, SINCERE.measurement, Literal(measurement, datatype=XSD.string))) + g.add((measurement_uri, SINCERE.time, Literal(time, datatype=XSD.string))) + g.add((measurement_uri, SINCERE.value, Literal(data_value, datatype=XSD.float))) + + # Serialize the graph in Turtle format and save to a file + g.serialize(destination=rdf_file_path, format="turtle") + + +# Function to read all JSON files from a folder and process each one +def process_json_files(input_dir, output_dir): + # Ensure the output directory exists + os.makedirs(output_dir, exist_ok=True) + + # Iterate over all files in the input directory + count = 1 + for file_name in os.listdir(input_dir): + file_path = os.path.join(input_dir, file_name) + + if os.path.isfile(file_path) and file_path.endswith('.json'): + try: + # Read JSON data from the file + with open(file_path, 'r') as file: + json_data = json.load(file) + + # Generate an RDF file path based on the file name + rdf_file_name = "output_channel_" + str(count) + ".ttl" + rdf_file_path = os.path.join(output_dir, rdf_file_name) + + # Convert JSON to RDF and save to file + json_to_rdf(json_data, rdf_file_path) + print(f"Processed {file_name} -> {rdf_file_path}") + count += 1 + except json.JSONDecodeError as e: + print(f"Error decoding JSON in file {file_name}: {e}") + except PermissionError: + print(f"Permission denied for file {file_name}") + + +# Example usage +input_dir = 'C:/Users/USER/PycharmProjects/sincere/meteorogical_data/input_json_file' # Replace with your input directory containing JSON files +output_dir = 'C:/Users/USER/PycharmProjects/sincere/meteorogical_data/output_json_file' # Replace with your desired output directory for RDF files + +process_json_files(input_dir, output_dir)