Skip to content

Use UUIDs for simulation ids. #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
andrewrosemberg merged 3 commits into from
Jul 18, 2023
Merged

Use UUIDs for simulation ids. #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
70ac975
change simulation ids to use UUIDs
andrewrosemberg Jul 18, 2023
9feb449
update readme
andrewrosemberg Jul 18, 2023
ae481a8
rm useless yml
andrewrosemberg Jul 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 0 additions & 47 deletions .travis.yml
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions Project.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
JuMP = "4076af6c-e467-56ae-b986-b466b2749572"
ParametricOptInterface = "0ce4ce61-57bf-432b-a095-efac525d185e"
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

[compat]
Arrow = "2"
Expand Down
13 changes: 7 additions & 6 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,11 @@ p = @variable(model, p in POI.Parameter(1.0)) # The parameter (defined using POI
@constraint(model, cons, x + p >= 3)
@objective(model, Min, 2x)

# The ids
problem_ids = collect(1:10)

# The parameter values
parameter_values = Dict(p => collect(1.0:10.0))

# The iterator
problem_iterator = ProblemIterator(problem_ids, parameter_values)
problem_iterator = ProblemIterator(parameter_values)
```

The parameter values of the problem iterator can be saved by simply:
Expand All @@ -52,6 +49,8 @@ Which creates the following CSV:
| 9 | 9.0 |
| 10 | 10.0|

ps.: For illustration purpose, I have represented the id's here as integers, but in reality they are generated as UUIDs.

### The Recorder

Then chose what values to record:
Expand Down Expand Up @@ -79,6 +78,8 @@ Which creates the following CSV:
| 9 | -6.0 | 2.0 |
| 10 | -7.0 | 2.0 |

ps.: Ditto id's.

Similarly, there is also the option to save the database in arrow files:

```julia
Expand All @@ -95,8 +96,8 @@ input_data = CSV.read("input_file.csv", DataFrame)
output_data = CSV.read("output_file.csv", DataFrame)

# Separate input and output variables
output_variables = output_data[:, 2:end]
input_features = input_data[output_data[:, 1], 2:end] # just use success solves
output_variables = output_data[!, Not(:id)]
input_features = innerjoin(input_data, output_data[!, [:id]], on = :id)[!, Not(:id)] # just use success solves

# Define model
model = Chain(
Expand Down
8 changes: 6 additions & 2 deletions examples/powermodels/flux_forecaster.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ function test_flux_forecaster(file_in::AbstractString, file_out::AbstractString)
output_data = CSV.read(file_out, DataFrame)

# Separate input and output variables
output_variables = output_data[:, 2:end]
input_features = input_data[output_data[:, 1], 2:end] # just use success solves
output_variables = output_data[!, Not(:id)]
input_features = innerjoin(input_data, output_data[!, [:id]], on = :id)[!, Not(:id)] # just use success solves

# Define model
model = Chain(
Expand All @@ -38,5 +38,9 @@ function test_flux_forecaster(file_in::AbstractString, file_out::AbstractString)
# Make predictions
predictions = model(input_features)
@test predictions isa Matrix

# Delete the files
rm(file_in)
rm(file_out)
end
end
1 change: 0 additions & 1 deletion examples/powermodels/pglib_datagen.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,6 @@ function generate_dataset_pglib(

# The problem iterator
problem_iterator = ProblemIterator(
collect(1:num_p),
Dict(
p .=> [
load_sampler(original_load[i], num_p) for
Expand Down
1 change: 1 addition & 0 deletions src/L2O.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module L2O
using Arrow
using CSV
using JuMP
using UUIDs
import ParametricOptInterface as POI
import Base: string

Expand Down
4 changes: 2 additions & 2 deletions src/arrowrecorder.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ abstract type ArrowFile <: RecorderFile end
Base.string(::Type{ArrowFile}) = "arrow"

"""
record(recorder::Recorder{ArrowFile}, model::JuMP.Model, id::T)
record(recorder::Recorder{ArrowFile}, id::UUID)

Record optimization problem solution to an Arrow file.
"""
function record(recorder::Recorder{ArrowFile}, model::JuMP.Model, id::T) where {T<:Integer}
function record(recorder::Recorder{ArrowFile}, id::UUID)
return Arrow.append(
recorder.filename,
(;
Expand Down
4 changes: 2 additions & 2 deletions src/csvrecorder.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ abstract type CSVFile <: RecorderFile end
Base.string(::Type{CSVFile}) = "csv"

"""
record(recorder::Recorder{CSVFile}, model::JuMP.Model, id::Int64)
record(recorder::Recorder{CSVFile}, id::UUID)

Record optimization problem solution to a CSV file.
"""
function record(recorder::Recorder{CSVFile}, model::JuMP.Model, id::Int64)
function record(recorder::Recorder{CSVFile}, id::UUID)
if !isfile(recorder.filename)
open(recorder.filename, "w") do f
write(f, "id")
Expand Down
24 changes: 17 additions & 7 deletions src/datasetgen.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,23 +22,33 @@ mutable struct Recorder{T<:RecorderFile}
end

"""
ProblemIterator(ids::Vector{Integer}, pairs::Dict{VariableRef, Vector{Real}})
ProblemIterator(ids::Vector{UUID}, pairs::Dict{VariableRef, Vector{Real}})

Iterator for optimization problem instances.
"""
struct ProblemIterator{T<:Real,Z<:Integer}
ids::Vector{Z}
struct ProblemIterator{T<:Real}
ids::Vector{UUID}
pairs::Dict{VariableRef,Vector{T}}
function ProblemIterator(
ids::Vector{Z}, pairs::Dict{VariableRef,Vector{T}}
) where {T<:Real,Z<:Integer}
ids::Vector{UUID}, pairs::Dict{VariableRef,Vector{T}}
) where {T<:Real}
for (p, val) in pairs
@assert length(ids) == length(val)
end
return new{T,Z}(ids, pairs)
return new{T}(ids, pairs)
end
end

function ProblemIterator(pairs::Dict{VariableRef,Vector{T}}) where {T<:Real}
ids = [uuid1() for _ in 1:length(first(values(pairs)))]
return ProblemIterator(ids, pairs)
end

"""
save(problem_iterator::ProblemIterator, filename::String, file_type::Type{T})

Save optimization problem instances to a file.
"""
function save(
problem_iterator::ProblemIterator, filename::String, file_type::Type{T}
) where {T<:RecorderFile}
Expand Down Expand Up @@ -85,7 +95,7 @@ function solve_and_record(
update_model!(model, problem_iterator.pairs, idx)
optimize!(model)
if recorder.filterfn(model)
record(recorder, model, problem_iterator.ids[idx])
record(recorder, problem_iterator.ids[idx])
return 1
end
return 0
Expand Down
4 changes: 2 additions & 2 deletions test/datasetgen.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@ function testdataset_gen(path::AbstractString)
# The problem iterator
num_p = 10
@test_throws AssertionError ProblemIterator(
collect(1:num_p), Dict(p => collect(1.0:3.0))
[uuid1() for _ in 1:num_p], Dict(p => collect(1.0:3.0))
)
@test_throws MethodError ProblemIterator(
collect(1.0:3.0), Dict(p => collect(1.0:3.0))
)
problem_iterator = ProblemIterator(collect(1:num_p), Dict(p => collect(1.0:num_p)))
problem_iterator = ProblemIterator(Dict(p => collect(1.0:num_p)))
file_input = joinpath(path, "test_input.$(string(filetype))") # file path
save(problem_iterator, file_input, filetype)
@test isfile(file_input)
Expand Down
10 changes: 6 additions & 4 deletions test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
using L2O
using Arrow
using Test
using DelimitedFiles
using JuMP, HiGHS
import ParametricOptInterface as POI
using Flux
using HiGHS
using JuMP
using L2O
import ParametricOptInterface as POI
using Test
using UUIDs

const test_dir = dirname(@__FILE__)
const examples_dir = joinpath(test_dir, "..", "examples")
Expand Down