Upload files to "meteorogical_data"
This commit is contained in:
97
meteorogical_data/device_data_rdf.py
Normal file
97
meteorogical_data/device_data_rdf.py
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
from rdflib import Graph, Namespace, URIRef, Literal, RDF, RDFS, XSD
|
||||||
|
from SPARQLWrapper import SPARQLWrapper, POST, BASIC, URLENCODED
|
||||||
|
|
||||||
|
class device():
|
||||||
|
|
||||||
|
|
||||||
|
def __init__(self, device_data_json, database, device_id):
|
||||||
|
self.device_data_json = device_data_json
|
||||||
|
self.database = database
|
||||||
|
self.device_id = device_id
|
||||||
|
|
||||||
|
def read_ttl_and_store_in_graphdb(self, ttl_file_path, graphdb_endpoint, graphdb_repository, username=None,
|
||||||
|
password=None):
|
||||||
|
"""
|
||||||
|
Reads RDF data from a Turtle file and stores it in a GraphDB repository.
|
||||||
|
|
||||||
|
:param ttl_file_path: Path to the Turtle file.
|
||||||
|
:param graphdb_endpoint: URL of the GraphDB endpoint.
|
||||||
|
:param graphdb_repository: Name of the GraphDB repository.
|
||||||
|
:param username: (Optional) Username for GraphDB authentication.
|
||||||
|
:param password: (Optional) Password for GraphDB authentication.
|
||||||
|
"""
|
||||||
|
# Create a graph
|
||||||
|
graph = Graph()
|
||||||
|
|
||||||
|
# Parse the Turtle file
|
||||||
|
graph.parse(ttl_file_path, format="turtle")
|
||||||
|
|
||||||
|
# Serialize the graph to a string in N-Triples format
|
||||||
|
ntriples_data = graph.serialize(format='nt')
|
||||||
|
|
||||||
|
# If ntriples_data is bytes, decode it to string
|
||||||
|
if isinstance(ntriples_data, bytes):
|
||||||
|
ntriples_data = ntriples_data.decode('utf-8')
|
||||||
|
|
||||||
|
# Create a SPARQLWrapper instance for the GraphDB endpoint
|
||||||
|
sparql = SPARQLWrapper(f"{graphdb_endpoint}/repositories/{graphdb_repository}/statements")
|
||||||
|
sparql.setMethod(POST)
|
||||||
|
sparql.setRequestMethod(URLENCODED)
|
||||||
|
sparql.addParameter('update', f"""
|
||||||
|
INSERT DATA {{
|
||||||
|
{ntriples_data}
|
||||||
|
}}
|
||||||
|
""")
|
||||||
|
|
||||||
|
# Set credentials if provided
|
||||||
|
if username and password:
|
||||||
|
sparql.setHTTPAuth(BASIC)
|
||||||
|
sparql.setCredentials(username, password)
|
||||||
|
|
||||||
|
# Send the request to the GraphDB repository
|
||||||
|
sparql.query()
|
||||||
|
|
||||||
|
def parse_device_data(self):
|
||||||
|
# Create a new RDF graph
|
||||||
|
g = Graph()
|
||||||
|
# Define a custom namespace
|
||||||
|
ontology = Namespace("http://sincere.org/s1#")
|
||||||
|
resource = Namespace("http://sincere.org/s1/resource/")
|
||||||
|
g.bind('sincere', ontology)
|
||||||
|
|
||||||
|
g.add((resource["device_" + self.device_id], RDF.type, ontology.Device))
|
||||||
|
g.add((resource["device_" + self.device_id], ontology.database, Literal(self.database, datatype=XSD.string)))
|
||||||
|
g.add((resource["device_" + self.device_id], RDFS.label, Literal(self.device_id, datatype=XSD.string)))
|
||||||
|
|
||||||
|
for dev in self.device_data_json:
|
||||||
|
channel = dev["channel"]
|
||||||
|
channel_id = dev["channel_id"]
|
||||||
|
constant = dev["constant"]
|
||||||
|
device_id = dev["device_id"]
|
||||||
|
item = dev["item"]
|
||||||
|
unit = dev["unit"]
|
||||||
|
writable = dev["writable"]
|
||||||
|
g.add((resource["device_" + self.device_id], ontology.hasChannel, resource["channel_" + channel_id]))
|
||||||
|
g.add((resource["channel_" + channel_id], RDF.type, ontology.Channel))
|
||||||
|
g.add((resource["channel_" + channel_id], ontology.number, Literal(channel, datatype=XSD.integer)))
|
||||||
|
g.add((resource["channel_" + channel_id], ontology.number, Literal(channel, datatype=XSD.integer)))
|
||||||
|
g.add((resource["channel_" + channel_id], ontology.constant, Literal(constant, datatype=XSD.string)))
|
||||||
|
g.add((resource["channel_" + channel_id], RDFS.label, Literal(channel_id, datatype=XSD.string)))
|
||||||
|
g.add((resource["channel_" + channel_id], ontology.device_id, Literal(device_id, datatype=XSD.string)))
|
||||||
|
g.add((resource["channel_" + channel_id], ontology.item, Literal(item, datatype=XSD.string)))
|
||||||
|
g.add((resource["channel_" + channel_id], ontology.unit, Literal(unit, datatype=XSD.string)))
|
||||||
|
g.add((resource["channel_" + channel_id], ontology.writable, Literal(writable, datatype=XSD.boolean)))
|
||||||
|
|
||||||
|
# Create RDF triples
|
||||||
|
# Serialize and save the RDF graph to a TTL file
|
||||||
|
file_path = "output_device.ttl"
|
||||||
|
g.serialize(destination=file_path, format="turtle")
|
||||||
|
|
||||||
|
graphdb_endpoint = 'http://localhost:7200' # Replace with the URL of your GraphDB endpoint
|
||||||
|
graphdb_repository = 'weather_measurement' # Replace with the name of your GraphDB repository
|
||||||
|
username = 'admin' # Replace with your GraphDB username (if required)
|
||||||
|
password = 'root' # Replace with your GraphDB password (if required)
|
||||||
|
|
||||||
|
self.read_ttl_and_store_in_graphdb(file_path, graphdb_endpoint, graphdb_repository, username, password)
|
||||||
|
print("Data was mapped in GraphDB")
|
||||||
|
return 1
|
||||||
248
meteorogical_data/devices.json
Normal file
248
meteorogical_data/devices.json
Normal file
@@ -0,0 +1,248 @@
|
|||||||
|
{
|
||||||
|
"status": 200,
|
||||||
|
"data": {
|
||||||
|
"reference": "DAVIS_Meteo",
|
||||||
|
"timer_loop": 120,
|
||||||
|
"measurement": "algete_meteo",
|
||||||
|
"file": "devis",
|
||||||
|
"gateway": "10.130.20.182",
|
||||||
|
"available": null,
|
||||||
|
"last_access": "2023-11-29T09:43:58.464780Z",
|
||||||
|
"channels": [
|
||||||
|
{
|
||||||
|
"channel": 48,
|
||||||
|
"item": "ForecastIcons",
|
||||||
|
"unit": "None",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "0a0332f1-8a21-49fc-99e9-cf2807a1b4ce"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 49,
|
||||||
|
"item": "ForecastRuleNumber",
|
||||||
|
"unit": "None",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "809e5a8a-13f9-418d-8578-a66e008602be"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 50,
|
||||||
|
"item": "TimeSunrise",
|
||||||
|
"unit": "HHMM",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "47c3fce8-ca58-4849-9d43-99164fd9cd0f"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 51,
|
||||||
|
"item": "TimeSunset",
|
||||||
|
"unit": "HHMM",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "a8c1479c-990f-4b9c-a0f0-bfd708ae010f"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 52,
|
||||||
|
"item": "WetBulb",
|
||||||
|
"unit": "HHMM",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "8c3a9ef7-4913-4630-bd3c-fbf3e0188592"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 62,
|
||||||
|
"item": "DewPoint",
|
||||||
|
"unit": "°F",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "fccfc8f9-b58a-495c-8cf3-817f5c6c439c"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 63,
|
||||||
|
"item": "HeatIndex",
|
||||||
|
"unit": "°F",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "a894273e-d703-4ed0-bd50-e80617d19e77"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 64,
|
||||||
|
"item": "WindChill",
|
||||||
|
"unit": "°F",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "a4f8ede7-7579-4dce-a812-bfa837d1ca1b"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 65,
|
||||||
|
"item": "THSWIndex",
|
||||||
|
"unit": "°F",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "3dc3baff-520c-4f50-a222-cffeb4794dda"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 66,
|
||||||
|
"item": "Last15minRain",
|
||||||
|
"unit": "Inches",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "62666981-2853-4117-9932-fa8104a4ab91"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 67,
|
||||||
|
"item": "LastHourRain",
|
||||||
|
"unit": "Inches",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "3399e58e-9857-466c-969c-14a0717b56c4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 68,
|
||||||
|
"item": "Last24HourRain",
|
||||||
|
"unit": "Inches",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "7b1caebe-b42c-49f7-854e-2fac8f010603"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 72,
|
||||||
|
"item": "BarometricSensorRawReading",
|
||||||
|
"unit": "Inches",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "c99843be-aa3d-4f0a-886f-dfbe54f90877"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 73,
|
||||||
|
"item": "AbsoluteBarometricPressure",
|
||||||
|
"unit": "Inches",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "eaec7c4c-e9ba-4865-8b65-9bc57077dead"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 2,
|
||||||
|
"item": "Barometer",
|
||||||
|
"unit": "inHg",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "4874b71d-2084-4967-a556-84c8772b5d18"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 3,
|
||||||
|
"item": "InsideTemperature",
|
||||||
|
"unit": "Cº",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "3450f1e5-f7ab-45a4-a0f6-4c43cc6fceaa"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 4,
|
||||||
|
"item": "InsideHumidity",
|
||||||
|
"unit": "%",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "bc9d8558-3268-484b-b96e-b161a587777d"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 5,
|
||||||
|
"item": "OutsideTemperature",
|
||||||
|
"unit": "None",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "f6c8f25a-ef51-4e5a-a99e-f4c544675e1d"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 6,
|
||||||
|
"item": "WindSpeed",
|
||||||
|
"unit": "None",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "3bf6013d-04f6-4601-8173-9ead59ed08ab"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 7,
|
||||||
|
"item": "10MinAvrgWindSpeed",
|
||||||
|
"unit": "None",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "178c0dba-25d6-41ac-93db-6b43520b5756"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 8,
|
||||||
|
"item": "WindDirection",
|
||||||
|
"unit": "None",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "64695b1a-960d-4ae9-a47b-3e5575070d20"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 17,
|
||||||
|
"item": "OutsideHumidity",
|
||||||
|
"unit": "None",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "17faa574-9935-4db1-a239-8efbb2653b3e"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 22,
|
||||||
|
"item": "RainRate",
|
||||||
|
"unit": "None",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "d267c1a4-efbc-4cd1-bb7b-98c6dc70d6e8"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 23,
|
||||||
|
"item": "UVIndex",
|
||||||
|
"unit": "None",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "eb2f0322-a921-4d73-98a4-b6bc66a8dceb"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 24,
|
||||||
|
"item": "SolarRadiation",
|
||||||
|
"unit": "W/m2",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "8a4c6b58-07e1-4cd5-9556-2fd7e75e6449"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"channel": 25,
|
||||||
|
"item": "StormRain",
|
||||||
|
"unit": "Inches",
|
||||||
|
"constant": null,
|
||||||
|
"writable": false,
|
||||||
|
"device_id": "372be277-4720-41e1-9e1a-e0d4fec2ee6e",
|
||||||
|
"channel_id": "db36ca06-d5a9-4bab-821a-4db134c212c1"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
73
meteorogical_data/helper_meteo.py
Normal file
73
meteorogical_data/helper_meteo.py
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
import json
|
||||||
|
from rdflib import Graph, Literal, Namespace, RDF, URIRef, XSD
|
||||||
|
import os
|
||||||
|
|
||||||
|
# Define the namespaces
|
||||||
|
SINCERE = Namespace("http://sincere.org/s1#")
|
||||||
|
|
||||||
|
|
||||||
|
# Function to convert JSON to RDF
|
||||||
|
def json_to_rdf(json_data, rdf_file_path):
|
||||||
|
# Create an RDF graph
|
||||||
|
g = Graph()
|
||||||
|
|
||||||
|
# Bind the SINCERE namespace
|
||||||
|
g.bind("sincere", SINCERE)
|
||||||
|
|
||||||
|
# Process each entry in the data
|
||||||
|
for i, entry in enumerate(json_data['data'], start=1):
|
||||||
|
channel_id = entry['channel_id']
|
||||||
|
data_value = entry['data_value']
|
||||||
|
measurement = entry['measurement']
|
||||||
|
time = entry['time']
|
||||||
|
|
||||||
|
channel_uri = URIRef(f"http://sincere.org/s1/resource/channel_{channel_id}")
|
||||||
|
measurement_uri = URIRef(f"http://sincere.org/s1/resource/measurement_{channel_id}_{i}")
|
||||||
|
|
||||||
|
g.add((channel_uri, RDF.type, SINCERE.Channel))
|
||||||
|
g.add((channel_uri, SINCERE.hasMeasurement, measurement_uri))
|
||||||
|
|
||||||
|
g.add((measurement_uri, RDF.type, SINCERE.Measurement))
|
||||||
|
g.add((measurement_uri, SINCERE.measurement, Literal(measurement, datatype=XSD.string)))
|
||||||
|
g.add((measurement_uri, SINCERE.time, Literal(time, datatype=XSD.string)))
|
||||||
|
g.add((measurement_uri, SINCERE.value, Literal(data_value, datatype=XSD.float)))
|
||||||
|
|
||||||
|
# Serialize the graph in Turtle format and save to a file
|
||||||
|
g.serialize(destination=rdf_file_path, format="turtle")
|
||||||
|
|
||||||
|
|
||||||
|
# Function to read all JSON files from a folder and process each one
|
||||||
|
def process_json_files(input_dir, output_dir):
|
||||||
|
# Ensure the output directory exists
|
||||||
|
os.makedirs(output_dir, exist_ok=True)
|
||||||
|
|
||||||
|
# Iterate over all files in the input directory
|
||||||
|
count = 1
|
||||||
|
for file_name in os.listdir(input_dir):
|
||||||
|
file_path = os.path.join(input_dir, file_name)
|
||||||
|
|
||||||
|
if os.path.isfile(file_path) and file_path.endswith('.json'):
|
||||||
|
try:
|
||||||
|
# Read JSON data from the file
|
||||||
|
with open(file_path, 'r') as file:
|
||||||
|
json_data = json.load(file)
|
||||||
|
|
||||||
|
# Generate an RDF file path based on the file name
|
||||||
|
rdf_file_name = "output_channel_" + str(count) + ".ttl"
|
||||||
|
rdf_file_path = os.path.join(output_dir, rdf_file_name)
|
||||||
|
|
||||||
|
# Convert JSON to RDF and save to file
|
||||||
|
json_to_rdf(json_data, rdf_file_path)
|
||||||
|
print(f"Processed {file_name} -> {rdf_file_path}")
|
||||||
|
count += 1
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
print(f"Error decoding JSON in file {file_name}: {e}")
|
||||||
|
except PermissionError:
|
||||||
|
print(f"Permission denied for file {file_name}")
|
||||||
|
|
||||||
|
|
||||||
|
# Example usage
|
||||||
|
input_dir = 'C:/Users/USER/PycharmProjects/sincere/meteorogical_data/input_json_file' # Replace with your input directory containing JSON files
|
||||||
|
output_dir = 'C:/Users/USER/PycharmProjects/sincere/meteorogical_data/output_json_file' # Replace with your desired output directory for RDF files
|
||||||
|
|
||||||
|
process_json_files(input_dir, output_dir)
|
||||||
Reference in New Issue
Block a user