Skip to content

Remove unused trip IDs ints. #31

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mmathieum merged 10 commits into from
Feb 3, 2026
Merged

Remove unused trip IDs ints. #31

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
23ab23e
Remove unused trip IDs ints.
mmathieum Feb 3, 2026
7057c06
fix
mmathieum Feb 3, 2026
b7e77aa
fix
mmathieum Feb 3, 2026
8ed1815
PR comments
mmathieum Feb 3, 2026
2e30cc1
PR comments
mmathieum Feb 3, 2026
8b911e5
cleanup
mmathieum Feb 3, 2026
73616a1
cleanup
mmathieum Feb 3, 2026
52c821f
PR comment
mmathieum Feb 3, 2026
ccf5742
Merge branch 'master' into mm/remove_unused_trip_ids
mmathieum Feb 3, 2026
312c5fb
Merge branch 'master' into mm/remove_unused_trip_ids
mmathieum Feb 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/main/java/org/mtransit/parser/gtfs/data/GSpec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,7 @@ public Collection<GStop> getAllStops() {
return this.stopsCache.values();
}

@SuppressWarnings("unused")
@NotNull
private Collection<String> getAllStopIds() {
if (USE_DB_ONLY) {
Expand Down
36 changes: 25 additions & 11 deletions src/main/java/org/mtransit/parser/mt/MGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.mtransit.parser.gtfs.GAgencyTools;
import org.mtransit.parser.gtfs.GReader;
import org.mtransit.parser.gtfs.data.GFieldTypes;
import org.mtransit.parser.gtfs.data.GIDs;
import org.mtransit.parser.gtfs.data.GSpec;
import org.mtransit.parser.mt.data.MAgency;
import org.mtransit.parser.mt.data.MFrequency;
Expand Down Expand Up @@ -80,8 +81,8 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age
HashSet<MRoute> mRoutes = new HashSet<>(); // use set to avoid duplicates
HashSet<MDirection> mDirections = new HashSet<>(); // use set to avoid duplicates
HashSet<MDirectionStop> mDirectionStops = new HashSet<>(); // use set to avoid duplicates
HashSet<MTrip> mTrips = new HashSet<>(); // use set to avoid duplicates
HashMap<Integer, MStop> mStops = new HashMap<>();
HashMap<Integer, MTrip> mTrips = new HashMap<>(); // map key unique -> avoid duplicates
HashMap<Integer, MStop> mStops = new HashMap<>(); // map key unique -> avoid duplicates
TreeMap<Long, List<MFrequency>> mRouteFrequencies = new TreeMap<>();
HashSet<MServiceDate> mServiceDates = new HashSet<>(); // use set to avoid duplicates
long firstTimestamp = -1L;
Expand All @@ -107,16 +108,21 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age
mRoutes.addAll(mRouteSpec.getRoutes());
mDirections.addAll(mRouteSpec.getDirections());
mDirectionStops.addAll(mRouteSpec.getDirectionStops());
mTrips.addAll(mRouteSpec.getTrips());
logMerging("trips...", mRouteId);
for (MTrip mTrip : mRouteSpec.getTrips()) {
final MTrip existing = mTrips.putIfAbsent(mTrip.getTripIdInt(), mTrip);
if (existing != null && !existing.equals(mTrip)) {
//noinspection DiscouragedApi
MTLog.log("%s: Trip ID '%s' already in list! (%s instead of %s)", mRouteId, mTrip.getTripId(), existing.toStringPlus(), mTrip.toStringPlus());
}
}
logMerging("trips... DONE", mRouteId);
logMerging("stops...", mRouteId);
for (MStop mStop : mRouteSpec.getStops()) {
if (mStops.containsKey(mStop.getId())) {
if (!mStops.get(mStop.getId()).equals(mStop)) {
MTLog.log("%s: Stop ID '%s' already in list! (%s instead of %s)", mRouteId, mStop.getId(), mStops.get(mStop.getId()), mStop);
}
continue;
final MStop existing = mStops.putIfAbsent(mStop.getId(), mStop);
if (existing != null && !existing.equals(mStop)) {
MTLog.log("%s: Stop ID '%s' already in list! (%s instead of %s)", mRouteId, mStop.getId(), existing, mStop);
}
mStops.put(mStop.getId(), mStop);
}
logMerging("stops... DONE", mRouteId);
logMerging("service dates...", mRouteId);
Expand Down Expand Up @@ -164,13 +170,21 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age
}
MTLog.log("Generating routes, trips, trip stops & stops objects... (all routes completed)");
threadPoolExecutor.shutdown();
MTLog.log("Removing unused IDs...");
int r;
try {
r = MTripIds.prune(new HashSet<>(GIDs.getStrings(mTrips.keySet())));
} catch (Exception e) {
throw new MTLog.Fatal(e, "Error while removing unused IDs!");
}
MTLog.log("Removing unused IDs... DONE (%d removed objects)", r);
final ArrayList<MAgency> mAgenciesList = new ArrayList<>(mAgencies);
Collections.sort(mAgenciesList);
final ArrayList<MStop> mStopsList = new ArrayList<>(mStops.values());
Collections.sort(mStopsList);
final ArrayList<MRoute> mRoutesList = new ArrayList<>(mRoutes);
Collections.sort(mRoutesList);
final ArrayList<MTrip> mTripsList = new ArrayList<>(mTrips);
final ArrayList<MTrip> mTripsList = new ArrayList<>(mTrips.values());
Collections.sort(mTripsList);
final ArrayList<MDirection> mDirectionsList = new ArrayList<>(mDirections);
Collections.sort(mDirectionsList);
Expand Down Expand Up @@ -629,7 +643,7 @@ private static void dumpTripIds(
dbStatement = dbConnection.createStatement();
sqlInsert = GTFSCommons.getT_TRIP_IDS_SQL_INSERT();
}
for (MTripId mTripId : MTripIds.getAll()) {
for (MTripId mTripId : MTripIds.getAllSorted()) {
final String tripIdsInsert = mTripId.toFile();
if (FeatureFlags.F_PRE_FILLED_DB) {
SQLUtils.executeUpdate(
Expand Down
33 changes: 31 additions & 2 deletions src/main/java/org/mtransit/parser/mt/data/MTripIds.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,32 @@ object MTripIds {
}
}

@JvmStatic
fun prune(usedTripIds: Set<String>): Int {
var removedCount = 0
synchronized(incrementLock) {
val iterator = idToIdInt.asMutableMap().entries.iterator()
while (iterator.hasNext()) {
val (tripId,tripIdInt) = iterator.next()
if (tripId !in usedTripIds) {
idIntToId.remove(tripIdInt)
iterator.remove()
removedCount++
}
}
}
return removedCount
}

@Suppress("unused")
@JvmStatic
fun remove(tripId: MTripId) {
synchronized(incrementLock) {
idIntToId.remove(tripId.tripIdInt)
idToIdInt.remove(tripId.tripId)
}
}

@Suppress("unused")
@JvmStatic
fun getId(tripIdInt: Int) =
Expand All @@ -57,9 +83,12 @@ object MTripIds {
idToIdInt.forEach { id, idInt ->
add(MTripId(idInt, id))
}
}.sorted()
}

@JvmStatic
fun getAllSorted() = getAll().sorted()

@JvmStatic
@JvmStatic
fun convert(tripId: String, quotesString: Boolean = false) =
if (FeatureFlags.F_EXPORT_TRIP_ID_INTS) {
getInt(tripId).toString()
Expand Down
Loading