PowerModels example

.gitignore

@@ -5,3 +5,6 @@
 /docs/build/
 Manifest.toml
 /.vscode/*
+*.arrow
+*.m
+*.csv

Project.toml

@@ -9,12 +9,17 @@ JuMP = "4076af6c-e467-56ae-b986-b466b2749572"
 ParametricOptInterface = "0ce4ce61-57bf-432b-a095-efac525d185e"
 
 [compat]
-julia = "1"
+JuMP = "1"
+Arrow = "2"
+ParametricOptInterface = "0.5"
+julia = "1.6"
 
 [extras]
 DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
+Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 HiGHS = "87dc4568-4c63-4d18-b0c0-bb2238e4078b"
+PowerModels = "c36e90e8-916a-50a6-bd94-075b64ef4655"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test", "DelimitedFiles", "HiGHS"]
+test = ["Test", "DelimitedFiles", "Downloads", "HiGHS", "PowerModels"]

docs/make.jl

@@ -14,12 +14,7 @@ makedocs(;
         edit_link="main",
         assets=String[],
     ),
-    pages=[
-        "Home" => "index.md",
-    ],
+    pages=["Home" => "index.md"],
 )
 
-deploydocs(;
-    repo="github.com/andrewrosemberg/L2O.jl",
-    devbranch="main",
-)
+deploydocs(; repo="github.com/andrewrosemberg/L2O.jl", devbranch="main")

examples/powermodels/pg_lib.jl

@@ -0,0 +1,79 @@
+using Downloads
+using PowerModels
+using JuMP, HiGHS
+import ParametricOptInterface as POI
+
+"""
+    createvarrefs!(sp::JuMP.Model, pm::AbstractPowerModel)
+
+create ref for anonimous variables on model
+"""
+function createvarrefs!(sp::JuMP.Model, pm::AbstractPowerModel)
+    for listvarref in values(PowerModels.var(pm))
+        for variableref in values(listvarref)
+            if typeof(variableref) == JuMP.VariableRef
+                sp[Symbol(name(variableref))] = variableref
+            end
+        end
+    end
+end
+
+"""
+    generate_dataset_pglib(data_dir::AbstractString, case_name::AbstractString; download_files::Bool=true, filetype::Type{RecorderFile},
+    num_p::Int=10
+)
+
+Generate dataset for pglib case_name with num_p problems and save it in data_dir
+"""
+function generate_dataset_pglib(
+    data_dir,
+    case_name;
+    filetype=CSVFile,
+    download_files=true,
+    num_p=10,
+)
+    case_file_path = joinpath(data_dir, case_name)
+    if download_files && !isfile(case_file_path)
+        Downloads.download(
+            "https://github.com/power-grid-lib/pglib-opf/01681386d084d8bd03b429abcd1ee6966f68b9a3/" *
+            case_name,
+            case_file_path,
+        )
+    end
+
+    # Read data
+    network_data = PowerModels.parse_file(case_file_path)
+
+    # The problem to iterate over
+    model = Model(() -> POI.Optimizer(HiGHS.Optimizer()))
+
+    # Save original load value and Link POI
+    original_load = network_data["load"]["1"]["pd"]
+    network_data["load"]["1"]["pd"] = p = @variable(model, _p in POI.Parameter(1.0))
+
+    # Instantiate the model
+    pm = instantiate_model(
+        network_data,
+        DCPPowerModel,
+        PowerModels.build_opf;
+        setting=Dict("output" => Dict("duals" => true)),
+        jump_model=model,
+    )
+
+    # The problem iterator
+    problem_iterator = ProblemIterator(
+        collect(1:num_p), Dict(p => collect(1.0:num_p) .* original_load)
+    )
+
+    # Create ref for anonimous variables on model
+    createvarrefs!(model, pm)
+
+    # Solve the problem and return the number of successfull solves
+    file = joinpath(data_dir, "test.$(string(filetype))")
+    number_generators = length(network_data["gen"])
+    recorder = Recorder{filetype}(
+        file; primal_variables=[Symbol("0_pg[$i]") for i in 1:number_generators]
+    )
+    return solve_batch(model, problem_iterator, recorder), number_generators
+end
+

src/L2O.jl

@@ -7,7 +7,6 @@ import Base: string
 
 export ArrowFile, CSVFile, ProblemIterator, Recorder, solve_batch
 
-
 include("datasetgen.jl")
 include("csvrecorder.jl")
 include("arrowrecorder.jl")

src/arrowrecorder.jl

@@ -8,23 +8,24 @@ Base.string(::Type{ArrowFile}) = "arrow"
 Record optimization problem solution to an Arrow file.
 """
 function record(recorder::Recorder{ArrowFile}, model::JuMP.Model, id::T) where {T<:Integer}
-    if !isfile(recorder.filename)
-        Arrow.append(
-            recorder.filename, (;
-                id = T[], 
-                zip(recorder.primal_variables, fill(Float64[], length(recorder.primal_variables)))..., 
-                zip(Symbol.("dual_" .* string.(recorder.dual_variables)), fill(Float64[], length(recorder.dual_variables)))..., 
-            )
-        )
-    end
-
-    Arrow.append(
-        recorder.filename, (;
-            id = [id], 
-            zip(recorder.primal_variables, [[MOI.get(model, MOI.VariablePrimal(), model[p])] for p in recorder.primal_variables])..., 
-            zip(Symbol.("dual_" .* string.(recorder.dual_variables)), [[MOI.get(model, MOI.ConstraintDual(), model[p])] for p in recorder.dual_variables])..., 
-        )
+    return Arrow.append(
+        recorder.filename,
+        (;
+            id=[id],
+            zip(
+                recorder.primal_variables,
+                [
+                    [MOI.get(model, MOI.VariablePrimal(), model[p])] for
+                    p in recorder.primal_variables
+                ],
+            )...,
+            zip(
+                Symbol.("dual_" .* string.(recorder.dual_variables)),
+                [
+                    [MOI.get(model, MOI.ConstraintDual(), model[p])] for
+                    p in recorder.dual_variables
+                ],
+            )...,
+        ),
     )
-
-
-end
+end

src/csvrecorder.jl

@@ -32,4 +32,4 @@ function record(recorder::Recorder{CSVFile}, model::JuMP.Model, id::Int64)
         end
         write(f, "\n")
     end
-end
+end

src/datasetgen.jl

@@ -11,7 +11,12 @@ mutable struct Recorder{T<:RecorderFile}
     dual_variables::AbstractArray{Symbol}
     filterfn::Function
 
-    function Recorder{T}(filename::String; primal_variables=[], dual_variables=[], filterfn=(model)-> termination_status(model) == MOI.OPTIMAL) where T<:RecorderFile
+    function Recorder{T}(
+        filename::String;
+        primal_variables=[],
+        dual_variables=[],
+        filterfn=(model) -> termination_status(model) == MOI.OPTIMAL,
+    ) where {T<:RecorderFile}
         return new{T}(filename, primal_variables, dual_variables, filterfn)
     end
 end
@@ -21,14 +26,16 @@ end
 
 Iterator for optimization problem instances.
 """
-struct ProblemIterator{T<:Real, Z<:Integer}
+struct ProblemIterator{T<:Real,Z<:Integer}
     ids::Vector{Z}
-    pairs::Dict{VariableRef, Vector{T}}
-    function ProblemIterator(ids::Vector{Z}, pairs::Dict{VariableRef, Vector{T}}) where {T<:Real, Z<:Integer}
+    pairs::Dict{VariableRef,Vector{T}}
+    function ProblemIterator(
+        ids::Vector{Z}, pairs::Dict{VariableRef,Vector{T}}
+    ) where {T<:Real,Z<:Integer}
         for (p, val) in pairs
             @assert length(ids) == length(val)
         end
-        return new{T, Z}(ids, pairs)
+        return new{T,Z}(ids, pairs)
     end
 end
 
@@ -37,17 +44,8 @@ end
 
 Update the value of a parameter in a JuMP model.
 """
-function update_model!(model::JuMP.Model, p::VariableRef, val::T) where {T<:Real}
-    MOI.set(model, POI.ParameterValue(), p, val)
-end
-
-"""
-    update_model!(model::JuMP.Model, p::VariableRef, val::AbstractArray{Real})
-
-Update the value of a parameter in a JuMP model.
-"""
-function update_model!(model::JuMP.Model, p::VariableRef, val::AbstractArray{T}) where {T<:Real}
-    MOI.set(model, POI.ParameterValue(), p, val)
+function update_model!(model::JuMP.Model, p::VariableRef, val)
+    return MOI.set(model, POI.ParameterValue(), p, val)
 end
 
 """
@@ -66,23 +64,28 @@ end
 
 Solve an optimization problem and record the solution.
 """
-function solve_and_record(model::JuMP.Model, problem_iterator::ProblemIterator, recorder::Recorder, idx::Integer)
+function solve_and_record(
+    model::JuMP.Model, problem_iterator::ProblemIterator, recorder::Recorder, idx::Integer
+)
     update_model!(model, problem_iterator.pairs, idx)
     optimize!(model)
     if recorder.filterfn(model)
         record(recorder, model, problem_iterator.ids[idx])
+        return 1
     end
-    return nothing
+    return 0
 end
 
 """
     solve_batch(model::JuMP.Model, problem_iterator::ProblemIterator, recorder::Recorder)
 
 Solve a batch of optimization problems and record the solutions.
 """
-function solve_batch(model::JuMP.Model, problem_iterator::ProblemIterator, recorder::Recorder)
-    for idx in 1:length(problem_iterator.ids)
-        solve_and_record(model, problem_iterator, recorder, idx)
-    end
-    return nothing
+function solve_batch(
+    model::JuMP.Model, problem_iterator::ProblemIterator, recorder::Recorder
+)
+    return sum(
+        solve_and_record(model, problem_iterator, recorder, idx) for
+        idx in 1:length(problem_iterator.ids)
+    ) / length(problem_iterator.ids)
 end

test/runtests.jl

@@ -5,23 +5,46 @@ using DelimitedFiles
 using JuMP, HiGHS
 import ParametricOptInterface as POI
 
-function testdataset_gen(path)
-    @testset "Dataset generation: $filetype" for filetype in [CSVFile, ArrowFile]
+include(
+    joinpath(
+        dirname(dirname(@__FILE__)), "examples", "powermodels", "pg_lib.jl"
+    ),
+)
+
+"""
+    testdataset_gen(path::String)
+
+Test dataset generation for different filetypes
+"""
+function testdataset_gen(path::String)
+    @testset "Type: $filetype" for filetype in [CSVFile, ArrowFile]
+        # The problem to iterate over
         model = Model(() -> POI.Optimizer(HiGHS.Optimizer()))
         @variable(model, x)
         p = @variable(model, _p in POI.Parameter(1.0))
         @constraint(model, cons, x + _p >= 3)
         @objective(model, Min, 2x)
 
+        # The problem iterator
         num_p = 10
+        @test_throws AssertionError ProblemIterator(collect(1:num_p), Dict(p => collect(1.0:3.0)))
+        @test_throws MethodError ProblemIterator(collect(1.0:3.0), Dict(p => collect(1.0:3.0)))
         problem_iterator = ProblemIterator(collect(1:num_p), Dict(p => collect(1.0:num_p)))
-        file = joinpath(path, "test.$(string(filetype))")
-        recorder = Recorder{filetype}(file, primal_variables=[:x], dual_variables=[:cons])
+
+        # The recorder
+        file = joinpath(path, "test.$(string(filetype))") # file path
+        @test Recorder{filetype}(file; primal_variables=[:x]) isa Recorder{filetype}
+        @test Recorder{filetype}(file; dual_variables=[:cons]) isa Recorder{filetype}
+        recorder = Recorder{filetype}(file; primal_variables=[:x], dual_variables=[:cons])
+
+        # Solve all problems and record solutions
         solve_batch(model, problem_iterator, recorder)
+
+        # Check if file exists and has the correct number of rows and columns
         if filetype == CSVFile
             file1 = joinpath(path, "test.csv")
             @test isfile(file1)
-            @test length(readdlm(file1, ',')[:, 1]) == num_p+1
+            @test length(readdlm(file1, ',')[:, 1]) == num_p + 1
             @test length(readdlm(file1, ',')[1, :]) == 3
             rm(file1)
         else
@@ -35,7 +58,30 @@ function testdataset_gen(path)
 end
 
 @testset "L2O.jl" begin
-    mktempdir() do path
-        testdataset_gen(path)
+    @testset "Dataset Generation" begin
+        mktempdir() do path
+            # Different filetypes
+            testdataset_gen(path)
+            # Pglib
+            @testset "pg_lib case" begin
+                # Define test case from pglib
+                case_name = "pglib_opf_case5_pjm.m"
+
+                # Define number of problems
+                num_p = 10
+
+                # Generate dataset
+                success_solves, number_generators = generate_dataset_pglib(
+                    path, case_name; num_p=num_p
+                )
+
+                # Check if the number of successfull solves is equal to the number of problems saved
+                file = joinpath(path, "test.csv")
+                @test isfile(file)
+                @test length(readdlm(file, ',')[:, 1]) == num_p * success_solves + 1
+                @test length(readdlm(file, ',')[1, :]) == number_generators + 1
+                rm(file)
+            end
+        end
     end
 end