From 741836a95342f583e80b9f252bd7fcb113887d91 Mon Sep 17 00:00:00 2001 From: Paul Em Date: Sun, 15 Oct 2023 00:08:23 +0200 Subject: [PATCH 1/4] setup basic familienkarte event collector --- .../eventcollector/BoudiccaEventCollectors.kt | 41 +++++------ .../collectors/FamilienkarteEventCollector.kt | 71 +++++++++++++++++++ 2 files changed, 92 insertions(+), 20 deletions(-) create mode 100644 eventcollectors/src/main/kotlin/events/boudicca/eventcollector/collectors/FamilienkarteEventCollector.kt diff --git a/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/BoudiccaEventCollectors.kt b/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/BoudiccaEventCollectors.kt index 134c777d..4e7b745d 100644 --- a/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/BoudiccaEventCollectors.kt +++ b/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/BoudiccaEventCollectors.kt @@ -7,25 +7,26 @@ fun main() { Thread.sleep(10000) // let eventdb startup first when both are deployed.... we should do a better thing here EventCollectorScheduler() .startWebUi() - .addEventCollector(LinzTermineCollector()) - .addEventCollector(PosthofCollector()) - .addEventCollector(JkuEventCollector()) - .addEventCollector(TechnologiePlauscherlCollector()) - .addEventCollector(ZuckerfabrikCollector()) - .addEventCollector(PlanetTTCollector()) - .addEventCollector(BrucknerhausCollector()) - .addEventCollector(OOESeniorenbundCollector()) - .addEventCollector(KupfTicketCollector()) - .addEventCollector(SpinnereiCollector()) - .addEventCollector(SchlachthofCollector()) - .addEventCollector(WissensturmCollector()) - .addEventCollector(LandestheaterLinzCollector()) - .addEventCollector(KapuCollector()) - .addEventCollector(StadtwerkstattCollector()) - .addEventCollector(InnovationsHauptplatzCodingWeeksCollector()) - .addEventCollector(OteloLinzCollector()) - .addEventCollector(EnnsEventsCollector()) - .addEventCollector(UlfOoeCollector()) - .addEventCollector(StiftskonzerteCollector()) +// .addEventCollector(LinzTermineCollector()) +// .addEventCollector(PosthofCollector()) +// .addEventCollector(JkuEventCollector()) +// .addEventCollector(TechnologiePlauscherlCollector()) +// .addEventCollector(ZuckerfabrikCollector()) +// .addEventCollector(PlanetTTCollector()) +// .addEventCollector(BrucknerhausCollector()) +// .addEventCollector(OOESeniorenbundCollector()) +// .addEventCollector(KupfTicketCollector()) +// .addEventCollector(SpinnereiCollector()) +// .addEventCollector(SchlachthofCollector()) +// .addEventCollector(WissensturmCollector()) +// .addEventCollector(LandestheaterLinzCollector()) +// .addEventCollector(KapuCollector()) +// .addEventCollector(StadtwerkstattCollector()) +// .addEventCollector(InnovationsHauptplatzCodingWeeksCollector()) +// .addEventCollector(OteloLinzCollector()) +// .addEventCollector(EnnsEventsCollector()) +// .addEventCollector(UlfOoeCollector()) +// .addEventCollector(StiftskonzerteCollector()) + .addEventCollector(FamilienkarteCollector()) .run() } \ No newline at end of file diff --git a/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/collectors/FamilienkarteEventCollector.kt b/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/collectors/FamilienkarteEventCollector.kt new file mode 100644 index 00000000..d38969d5 --- /dev/null +++ b/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/collectors/FamilienkarteEventCollector.kt @@ -0,0 +1,71 @@ +package events.boudicca.eventcollector.collectors + +import events.boudicca.SemanticKeys +import events.boudicca.api.eventcollector.Event +import events.boudicca.api.eventcollector.EventCollector +import events.boudicca.api.eventcollector.Fetcher +import events.boudicca.api.eventcollector.TwoStepEventCollector +import net.fortuna.ical4j.data.CalendarBuilder +import net.fortuna.ical4j.model.Calendar +import net.fortuna.ical4j.model.component.VEvent +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.net.URI +import java.net.URL +import java.time.* +import java.time.format.DateTimeFormatter + +class FamilienkarteCollector : TwoStepEventCollector("familienkarte") { + + private val fetcher = Fetcher() + + + override fun getAllUnparsedEvents(): List { + val document = Jsoup.parse(fetcher.fetchUrl("https://www.familienkarte.at/de/freizeit/veranstaltungen/veranstaltungskalender.html?events_cat_key=8")) + return document.select("div.detailButton a") + .map { it.attr("href") } + } + + override fun parseEvent(event: String): Event { + val eventSite = Jsoup.parse(fetcher.fetchUrl(event)) + + val name = eventSite.select("div.eventDetailWrapper h1").text() + val (startDate, endDate) = parseDates(eventSite) + + val data = mutableMapOf() + data[SemanticKeys.URL] = event + if (endDate != null) { + data[SemanticKeys.ENDDATE] = endDate.format(DateTimeFormatter.ISO_DATE) + } + data[SemanticKeys.TYPE] = "other" + data[SemanticKeys.DESCRIPTION] = eventSite.select("div.eventDetailDescr").text() + + val img = eventSite.select("div.eventEntry img") + if (!img.isEmpty()) { + data[SemanticKeys.PICTUREURL] = img.first()!!.attr("src") + } + + data[SemanticKeys.LOCATION_NAME] = eventSite.select("div.eventDetailLocation").text() + + return Event(name, startDate, data) + } + + + private fun parseDates(element: Element): Pair { + val fullDate = element.select("div.eventDetailWrapper h2").text() + val date = fullDate.substring(3, 11) + val localDate = LocalDate.parse(date, DateTimeFormatter.ofPattern("dd.MM.uu")) + + val startAndEndTimeText = fullDate.substring(14, 27) + val startAndEndTimes = startAndEndTimeText.split(" - ") + + val localStartTime = LocalTime.parse(startAndEndTimes[0].trim(), DateTimeFormatter.ofPattern("kk:mm")) + val localEndTime = LocalTime.parse(startAndEndTimes[1].trim(), DateTimeFormatter.ofPattern("kk:mm")) + + return Pair( + localDate.atTime(localStartTime).atZone(ZoneId.of("Europe/Vienna")).toOffsetDateTime(), + localDate.atTime(localEndTime).atZone(ZoneId.of("Europe/Vienna")).toOffsetDateTime(), + ) + } +} From b5640dfcb514bb5c099d21643f234b3e62fe417a Mon Sep 17 00:00:00 2001 From: Paul Em Date: Sun, 15 Oct 2023 00:33:11 +0200 Subject: [PATCH 2/4] adjust test case --- .../events/boudicca/eventcollector/LocalCollectorDebug.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/LocalCollectorDebug.kt b/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/LocalCollectorDebug.kt index f2013f9b..72cb1593 100644 --- a/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/LocalCollectorDebug.kt +++ b/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/LocalCollectorDebug.kt @@ -24,5 +24,6 @@ fun main() { // .debug(OteloLinzCollector()) // .debug(EnnsEventsCollector()) // .debug(UlfOoeCollector()) - .debug(StiftskonzerteCollector()) +// .debug(StiftskonzerteCollector()) + .debug(FamilienkarteCollector()) } \ No newline at end of file From 89c75fa29163b9bb194cdd021bf24cc8786266ee Mon Sep 17 00:00:00 2001 From: Paul Em Date: Sun, 15 Oct 2023 00:33:20 +0200 Subject: [PATCH 3/4] optimize imports --- .../collectors/FamilienkarteEventCollector.kt | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/collectors/FamilienkarteEventCollector.kt b/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/collectors/FamilienkarteEventCollector.kt index d38969d5..a1c0841a 100644 --- a/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/collectors/FamilienkarteEventCollector.kt +++ b/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/collectors/FamilienkarteEventCollector.kt @@ -2,18 +2,14 @@ package events.boudicca.eventcollector.collectors import events.boudicca.SemanticKeys import events.boudicca.api.eventcollector.Event -import events.boudicca.api.eventcollector.EventCollector import events.boudicca.api.eventcollector.Fetcher import events.boudicca.api.eventcollector.TwoStepEventCollector -import net.fortuna.ical4j.data.CalendarBuilder -import net.fortuna.ical4j.model.Calendar -import net.fortuna.ical4j.model.component.VEvent import org.jsoup.Jsoup -import org.jsoup.nodes.Document import org.jsoup.nodes.Element -import java.net.URI -import java.net.URL -import java.time.* +import java.time.LocalDate +import java.time.LocalTime +import java.time.OffsetDateTime +import java.time.ZoneId import java.time.format.DateTimeFormatter class FamilienkarteCollector : TwoStepEventCollector("familienkarte") { From 78997d74db17c6bb9221492afa562cace30e8c66 Mon Sep 17 00:00:00 2001 From: Paul Em Date: Sun, 15 Oct 2023 00:34:35 +0200 Subject: [PATCH 4/4] reset main collector --- .../eventcollector/BoudiccaEventCollectors.kt | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/BoudiccaEventCollectors.kt b/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/BoudiccaEventCollectors.kt index 4e7b745d..dba4edf1 100644 --- a/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/BoudiccaEventCollectors.kt +++ b/eventcollectors/src/main/kotlin/events/boudicca/eventcollector/BoudiccaEventCollectors.kt @@ -7,26 +7,26 @@ fun main() { Thread.sleep(10000) // let eventdb startup first when both are deployed.... we should do a better thing here EventCollectorScheduler() .startWebUi() -// .addEventCollector(LinzTermineCollector()) -// .addEventCollector(PosthofCollector()) -// .addEventCollector(JkuEventCollector()) -// .addEventCollector(TechnologiePlauscherlCollector()) -// .addEventCollector(ZuckerfabrikCollector()) -// .addEventCollector(PlanetTTCollector()) -// .addEventCollector(BrucknerhausCollector()) -// .addEventCollector(OOESeniorenbundCollector()) -// .addEventCollector(KupfTicketCollector()) -// .addEventCollector(SpinnereiCollector()) -// .addEventCollector(SchlachthofCollector()) -// .addEventCollector(WissensturmCollector()) -// .addEventCollector(LandestheaterLinzCollector()) -// .addEventCollector(KapuCollector()) -// .addEventCollector(StadtwerkstattCollector()) -// .addEventCollector(InnovationsHauptplatzCodingWeeksCollector()) -// .addEventCollector(OteloLinzCollector()) -// .addEventCollector(EnnsEventsCollector()) -// .addEventCollector(UlfOoeCollector()) -// .addEventCollector(StiftskonzerteCollector()) + .addEventCollector(LinzTermineCollector()) + .addEventCollector(PosthofCollector()) + .addEventCollector(JkuEventCollector()) + .addEventCollector(TechnologiePlauscherlCollector()) + .addEventCollector(ZuckerfabrikCollector()) + .addEventCollector(PlanetTTCollector()) + .addEventCollector(BrucknerhausCollector()) + .addEventCollector(OOESeniorenbundCollector()) + .addEventCollector(KupfTicketCollector()) + .addEventCollector(SpinnereiCollector()) + .addEventCollector(SchlachthofCollector()) + .addEventCollector(WissensturmCollector()) + .addEventCollector(LandestheaterLinzCollector()) + .addEventCollector(KapuCollector()) + .addEventCollector(StadtwerkstattCollector()) + .addEventCollector(InnovationsHauptplatzCodingWeeksCollector()) + .addEventCollector(OteloLinzCollector()) + .addEventCollector(EnnsEventsCollector()) + .addEventCollector(UlfOoeCollector()) + .addEventCollector(StiftskonzerteCollector()) .addEventCollector(FamilienkarteCollector()) .run() } \ No newline at end of file