diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java b/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java index 33d974f..7ef497c 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java +++ b/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java @@ -326,6 +326,7 @@ public Collection getAllStops() { return this.stopsCache.values(); } + @SuppressWarnings("unused") @NotNull private Collection getAllStopIds() { if (USE_DB_ONLY) { diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index 43c3ebc..8d0dfe3 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -23,6 +23,7 @@ import org.mtransit.parser.gtfs.GAgencyTools; import org.mtransit.parser.gtfs.GReader; import org.mtransit.parser.gtfs.data.GFieldTypes; +import org.mtransit.parser.gtfs.data.GIDs; import org.mtransit.parser.gtfs.data.GSpec; import org.mtransit.parser.mt.data.MAgency; import org.mtransit.parser.mt.data.MFrequency; @@ -80,8 +81,8 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age HashSet mRoutes = new HashSet<>(); // use set to avoid duplicates HashSet mDirections = new HashSet<>(); // use set to avoid duplicates HashSet mDirectionStops = new HashSet<>(); // use set to avoid duplicates - HashSet mTrips = new HashSet<>(); // use set to avoid duplicates - HashMap mStops = new HashMap<>(); + HashMap mTrips = new HashMap<>(); // map key unique -> avoid duplicates + HashMap mStops = new HashMap<>(); // map key unique -> avoid duplicates TreeMap> mRouteFrequencies = new TreeMap<>(); HashSet mServiceDates = new HashSet<>(); // use set to avoid duplicates long firstTimestamp = -1L; @@ -107,16 +108,21 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age mRoutes.addAll(mRouteSpec.getRoutes()); mDirections.addAll(mRouteSpec.getDirections()); mDirectionStops.addAll(mRouteSpec.getDirectionStops()); - mTrips.addAll(mRouteSpec.getTrips()); + logMerging("trips...", mRouteId); + for (MTrip mTrip : mRouteSpec.getTrips()) { + final MTrip existing = mTrips.putIfAbsent(mTrip.getTripIdInt(), mTrip); + if (existing != null && !existing.equals(mTrip)) { + //noinspection DiscouragedApi + MTLog.log("%s: Trip ID '%s' already in list! (%s instead of %s)", mRouteId, mTrip.getTripId(), existing.toStringPlus(), mTrip.toStringPlus()); + } + } + logMerging("trips... DONE", mRouteId); logMerging("stops...", mRouteId); for (MStop mStop : mRouteSpec.getStops()) { - if (mStops.containsKey(mStop.getId())) { - if (!mStops.get(mStop.getId()).equals(mStop)) { - MTLog.log("%s: Stop ID '%s' already in list! (%s instead of %s)", mRouteId, mStop.getId(), mStops.get(mStop.getId()), mStop); - } - continue; + final MStop existing = mStops.putIfAbsent(mStop.getId(), mStop); + if (existing != null && !existing.equals(mStop)) { + MTLog.log("%s: Stop ID '%s' already in list! (%s instead of %s)", mRouteId, mStop.getId(), existing, mStop); } - mStops.put(mStop.getId(), mStop); } logMerging("stops... DONE", mRouteId); logMerging("service dates...", mRouteId); @@ -164,13 +170,21 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age } MTLog.log("Generating routes, trips, trip stops & stops objects... (all routes completed)"); threadPoolExecutor.shutdown(); + MTLog.log("Removing unused IDs..."); + int r; + try { + r = MTripIds.prune(new HashSet<>(GIDs.getStrings(mTrips.keySet()))); + } catch (Exception e) { + throw new MTLog.Fatal(e, "Error while removing unused IDs!"); + } + MTLog.log("Removing unused IDs... DONE (%d removed objects)", r); final ArrayList mAgenciesList = new ArrayList<>(mAgencies); Collections.sort(mAgenciesList); final ArrayList mStopsList = new ArrayList<>(mStops.values()); Collections.sort(mStopsList); final ArrayList mRoutesList = new ArrayList<>(mRoutes); Collections.sort(mRoutesList); - final ArrayList mTripsList = new ArrayList<>(mTrips); + final ArrayList mTripsList = new ArrayList<>(mTrips.values()); Collections.sort(mTripsList); final ArrayList mDirectionsList = new ArrayList<>(mDirections); Collections.sort(mDirectionsList); @@ -629,7 +643,7 @@ private static void dumpTripIds( dbStatement = dbConnection.createStatement(); sqlInsert = GTFSCommons.getT_TRIP_IDS_SQL_INSERT(); } - for (MTripId mTripId : MTripIds.getAll()) { + for (MTripId mTripId : MTripIds.getAllSorted()) { final String tripIdsInsert = mTripId.toFile(); if (FeatureFlags.F_PRE_FILLED_DB) { SQLUtils.executeUpdate( diff --git a/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt index 15d5e4a..73af2ec 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt @@ -37,6 +37,32 @@ object MTripIds { } } + @JvmStatic + fun prune(usedTripIds: Set): Int { + var removedCount = 0 + synchronized(incrementLock) { + val iterator = idToIdInt.asMutableMap().entries.iterator() + while (iterator.hasNext()) { + val (tripId,tripIdInt) = iterator.next() + if (tripId !in usedTripIds) { + idIntToId.remove(tripIdInt) + iterator.remove() + removedCount++ + } + } + } + return removedCount + } + + @Suppress("unused") + @JvmStatic + fun remove(tripId: MTripId) { + synchronized(incrementLock) { + idIntToId.remove(tripId.tripIdInt) + idToIdInt.remove(tripId.tripId) + } + } + @Suppress("unused") @JvmStatic fun getId(tripIdInt: Int) = @@ -57,9 +83,12 @@ object MTripIds { idToIdInt.forEach { id, idInt -> add(MTripId(idInt, id)) } - }.sorted() + } + + @JvmStatic + fun getAllSorted() = getAll().sorted() - @JvmStatic + @JvmStatic fun convert(tripId: String, quotesString: Boolean = false) = if (FeatureFlags.F_EXPORT_TRIP_ID_INTS) { getInt(tripId).toString()