OGD made easy #2 - Datenintegration und -publikation

import ch.so.agi.gretl.tasks.*
import java.nio.file.Files
import java.nio.file.Paths
import de.undercouch.gradle.tasks.download.Download

apply plugin: 'ch.so.agi.gretl'

defaultTasks "runIntegrationJob"

def pathToTempFolder = System.getProperty("java.io.tmpdir")
def resultDir = file("./result")
resultDir.mkdirs()

def csvFileName = "ch.so.afu.abfallmengen_gemeinden.csv"
def baseName = "ch.so.afu.abfallmengen_gemeinden"
def csvFileObj = file(Paths.get(resultDir.toString(), csvFileName))
def parquetFileName = baseName + ".parquet"
def excelFileName = baseName + ".xlsx"
def tomlFileName = baseName + ".toml"
def bucket = "ch.so.data-dev"
def modelName = "SO_AFU_Abfallmengen_Gemeinden_20230629"
def defaultModelDir = projectDir.toString()+";https://geo.so.ch/models"

// Könnte auch Upload durch Benutzer sein.
tasks.register('downloadCsv', Download) {
    src "https://s3.eu-central-1.amazonaws.com/ch.so.data.ingress-demo/$baseName/$csvFileName"
    dest csvFileObj
    overwrite true

    doLast {
        println "File downloaded to: " + pathToTempFolder
    }
}

tasks.register('validateCsv', CsvValidator) {
    dependsOn 'downloadCsv'
    dataFiles = [csvFileObj.toString()]
    firstLineIsHeader = true
    valueDelimiter = null
    valueSeparator = ";"
    models = modelName
    modeldir = defaultModelDir
}

tasks.register('createParquet', Csv2Parquet) {
    dependsOn 'validateCsv'
    csvFile = csvFileObj
    firstLineIsHeader = true
    valueDelimiter = null
    valueSeparator = ";"
    models = modelName
    modeldir = defaultModelDir
    outputDir = file(resultDir)
}

tasks.register('createExcel', Csv2Excel) {
    dependsOn 'createParquet'
    csvFile = csvFileObj
    firstLineIsHeader = true
    valueDelimiter = null
    valueSeparator = ";"
    models = modelName
    modeldir = defaultModelDir
    outputDir = file(resultDir)
}

tasks.register('createMeta', OgdMetaPublisher) {
    dependsOn 'createExcel'
    configFile = file(tomlFileName)
    outputDir = resultDir
}

tasks.register('uploadFiles', S3Upload) {
    dependsOn 'createMeta'
    accessKey = awsAccessKeyAgi
    secretKey = awsSecretAccessKeyAgi
    sourceFiles = fileTree(resultDir) { include "*.parquet" include "*.xlsx" include "*.csv" include "*.xtf" }
    endPoint = "https://s3.eu-central-1.amazonaws.com"
    region = "eu-central-1"
    bucketName = bucket
    acl = "public-read"
}

tasks.register('runIntegrationJob') {
    dependsOn 'uploadFiles'
}

name: ogd-job
on:
  workflow_dispatch:
    inputs:
      version:
        description: 'identifier?'
        required: true

jobs:
  dataIntegration:
    env:
      ORG_GRADLE_PROJECT_awsAccessKeyAgi: ${{secrets.AWS_ACCESS_KEY_ID}}
      ORG_GRADLE_PROJECT_awsSecretAccessKeyAgi: ${{secrets.AWS_SECRET_ACCESS_KEY}}

    runs-on: ubuntu-latest

    container:
      image: sogis/gretl:latest

    steps:
      - uses: actions/checkout@v3

      - name: Run GRETL job
        run: |
          gradle -b ${{ github.event.inputs.version }}/build.gradle --init-script /home/gradle/init.gradle --no-daemon

IoxEvent event = xtfReader.read();
while (event instanceof IoxEvent) {
    if (event instanceof ObjectEvent) {
        ObjectEvent objectEvent = (ObjectEvent) event;
        IomObject iomObj = objectEvent.getIomObject();
        IomObject[] iomObjects = new IomObject[] {iomObj};
        Writer writer = new StringWriter();
        JsonGenerator jg = objectMapper.createGenerator(writer);
        Iox2jsonUtility.write(jg, iomObjects, td);
        jg.flush();
        jg.close();
        String jsonString = writer.toString();
        // do something with jsonString
    }
    event = xtfReader.read();
}

[
  {
    "@type": "SO_OGD_Metadata_20230629.Datasets.Dataset",
    "@id": "ch.so.agi.amtliche_vermessung_statistik",
    "Identifier": "ch.so.agi.amtliche_vermessung_statistik",
    "Title": "Statistische Kennzahlen der amtlichen Vermessung",
    "Description": "Statistische Kennzahlen der amtlichen Vermessung über Personal und Umsatz in den Jahren 1983 bis 2022.",
    "Publisher": {
      "@type": "SO_OGD_Metadata_20230629.Office_",
      "AgencyName": "Amt für Geoinformation",
      "Abbreviation": "AGI",
      "OfficeAtWeb": "https://agi.so.ch",
      "Email": "mailto:agi@bd.so.ch",
      "Phone": "032 627 75 92"
    },
    "Theme": "Statistik,Amtliche Vermessung",
    "Keywords": "Statistik,Amtliche Vermessung",
    "StartDate": "1983-01-01",
    "EndDate": "2022-12-31",
    "Resources": [
      {
        "@type": "SO_OGD_Metadata_20230629.Resource",
        "Identifier": "ch.so.agi.amtliche_vermessung_statistik.umsatz",
        "Title": "Umsatz pro Jahr",
        "Description": "Umsatz pro Jahr. Anzahl Gebäudemutationen und Grundstücksmutationen und Gesamtumsatz in Franken.",
        "Model": {
          "@type": "SO_OGD_Metadata_20230629.ModelLink",
          "Name": "SO_AGI_Amtliche_Vermessung_Statistik_Umsatz_20230625",
          "LocationHint": "https://geo.so.ch/models"
        },
     ....