Invariant naming of generators

docs/src/snoopi.md

@@ -200,3 +200,28 @@ end
 This covers `+`, `-`, `*`, `/`, and conversion for various combinations of types.
 The results from `@snoopi` can suggest method/type combinations that might be useful to
 precompile, but often you can generalize its suggestions in useful ways.
+
+## Analyzing omitted methods
+
+There are some methods that cannot be precompiled. For example, suppose you have two packages,
+`A` and `B`, that are independent of one another.
+Then `A.f([B.Object(1)])` cannot be precompiled, because `A` does not know about `B.Object`,
+and `B` does not know about `A.f`, unless both `A` and `B` get included into another package.
+
+Such problematic methods are removed automatically.
+If you want to be informed about these removals, you can use Julia's logging framework
+while running `parcel`:
+
+```
+julia> using Base.CoreLogging
+
+julia> logger = SimpleLogger(IOBuffer(), CoreLogging.Debug);
+
+julia> pc = with_logger(logger) do
+           SnoopCompile.parcel(inf_timing)
+       end
+
+julia> msgs = String(take!(logger.stream))
+```
+
+The omitted methods will be logged to the string `msgs`.

src/SnoopCompile.jl

@@ -177,8 +177,9 @@ function extract_topmod(e)
     return :unknown
 end
 
-const anonrex = r"#{1,2}\d+#{1,2}\d+"       # detect anonymous functions
-const kwrex = r"^#kw##(.*)$|#([^#]*)##kw$"  # detect keyword-supplying functions
+const anonrex = r"#{1,2}\d+#{1,2}\d+"         # detect anonymous functions
+const kwrex = r"^#kw##(.*)$|^#([^#]*)##kw$"   # detect keyword-supplying functions
+const genrex = r"^##s\d+#\d+$"                # detect generators for @generated functions
 
 function parse_call(line; subst=Vector{Pair{String, String}}(), blacklist=String[])
     match(anonrex, line) === nothing || return false, line, :unknown, ""
@@ -272,51 +273,109 @@ function topmodule(mods)
     return mod
 end
 
+function addmodules!(mods, parameters)
+    for p in parameters
+        if isa(p, DataType)
+            push!(mods, Base.moduleroot(p.name.module))
+            addmodules!(mods, p.parameters)
+        end
+    end
+    return mods
+end
+
+function methods_with_generators(m::Module)
+    meths = Method[]
+    for name in names(m; all=true)
+        isdefined(m, name) || continue
+        f = getfield(m, name)
+        if isa(f, Function)
+            for method in methods(f)
+                if isdefined(method, :generator)
+                    push!(meths, method)
+                end
+            end
+        end
+    end
+    return meths
+end
+
 function parcel(tinf::AbstractVector{Tuple{Float64,Core.MethodInstance}}; subst=Vector{Pair{String, String}}(), blacklist=String[])
-    pc = Dict{Symbol, Vector{String}}()
-    mods = Set{Module}()
+    pc = Dict{Symbol, Vector{String}}()      # output
+    modgens = Dict{Module, Vector{Method}}() # methods with generators in a module
+    mods = Set{Module}()                     # module of each parameter for a given method
     for (t, mi) in reverse(tinf)
         isdefined(mi, :specTypes) || continue
         tt = mi.specTypes
         m = mi.def
         isa(m, Method) || continue
+        # Determine which module to assign this method to. All the types in the arguments
+        # need to be defined; we collect all the corresponding modules and assign it to the
+        # "topmost".
         empty!(mods)
         push!(mods, Base.moduleroot(m.module))
-        ok = true
-        for p in tt.parameters
-            if isa(p, DataType)
-                if match(anonrex, String(p.name.name)) !== nothing
-                    ok = false
-                    break
-                end
-                push!(mods, Base.moduleroot(p.name.module))
-            end
-        end
-        ok || continue
+        addmodules!(mods, tt.parameters)
         topmod = topmodule(mods)
-        topmod === nothing && continue
-        paramrepr = map(tt.parameters) do p
-            mkw = match(kwrex, String(p.name.name))
-            if mkw !== nothing
-                fname = mkw.captures[1] === nothing ? mkw.captures[2] : mkw.captures[1]
-                thismod = p.name.module
-                "Core.kwftype(typeof($thismod.$fname))"
+        if topmod === nothing
+            @debug "Skipping $tt due to lack of a suitable top module"
+            continue
+        end
+        # If we haven't yet started the list for this module, initialize
+        topmodname = nameof(topmod)
+        if !haskey(pc, topmodname)
+            pc[topmodname] = String[]
+        end
+        # Create the string representation of the signature
+        # Use special care with keyword functions, anonymous functions
+        prefix = ""
+        p = tt.parameters[1]
+        mname, mmod = String(p.name.name), m.module   # m.name strips the kw identifier
+        mkw = match(kwrex, mname)
+        manon = match(anonrex, mname)
+        mgen = match(genrex, mname)
+        frepr = if mkw !== nothing
+            # Keyword function
+            fname = mkw.captures[1] === nothing ? mkw.captures[2] : mkw.captures[1]
+            "Core.kwftype(typeof($mmod.$fname))"
+        elseif manon !== nothing
+            # Anonymous function, wrap in an `isdefined`
+            prefix = "if isdefined($mmod, Symbol(\"$mname\")"
+            "getfield($mmod, Symbol(\"$mname\"))"  # this is universal, var is Julia 1.3+
+        elseif mgen !== nothing
+            # Generator for a @generated function
+            if !haskey(modgens, m.module)
+                callers = modgens[m.module] = methods_with_generators(m.module)
             else
-                repr(p)
+                callers = modgens[m.module]
+            end
+            getgen = "nothing"
+            for caller in callers
+                if nameof(caller.generator.gen) == m.name
+                    getgen = "typeof(first(whichtt($(caller.sig))).generator.gen)"
+                    break
+                end
             end
+            getgen
+        else
+            repr(p)
         end
+        paramrepr = map(repr, Iterators.drop(tt.parameters, 1))
+        pushfirst!(paramrepr, frepr)
         ttrepr = "Tuple{" * join(paramrepr, ',') * '}'
+        # Check that we parsed it correctly, and that all types are defined in the module
         ttexpr = Meta.parse(ttrepr)
-        try
-            Core.eval(topmod, ttexpr)
-        catch
-            continue
+        if mgen === nothing  # whichtt is not defined in topmod
+            try
+                Core.eval(topmod, ttexpr)
+            catch
+                @debug "Module $topmod: skipping $ttrepr due to eval failure"
+                continue
+            end
         end
-        topmodname = nameof(topmod)
-        if !haskey(pc, topmodname)
-            pc[topmodname] = String[]
+        stmt = "precompile(" * ttrepr * ')'
+        if !isempty(prefix)
+            stmt = prefix * ' ' * stmt * " end"
         end
-        push!(pc[topmodname], "precompile(" * ttrepr * ')')
+        push!(pc[topmodname], stmt)
     end
     return pc
 end
@@ -388,6 +447,15 @@ function write(prefix::AbstractString, pc::Dict; always::Bool = false)
     end
     for (k, v) in pc
         open(joinpath(prefix, "precompile_$k.jl"), "w") do io
+            println(io, """
+            # Like `which` but takes the full signature Tuple-type (including `typeof(f)`)
+            # Used to get the generator of @generated functions, if present
+            function whichtt(@nospecialize(tt))
+                m = ccall(:jl_gf_invoke_lookup, Any, (Any, UInt), tt, typemax(UInt))
+                m === nothing && return nothing
+                return m.func::Method
+            end
+            """)
             println(io, "function _precompile_()")
             !always && println(io, "    ccall(:jl_generating_output, Cint, ()) == 1 || return nothing")
             for ln in v

test/E.jl

@@ -3,4 +3,20 @@ module E
     struct ET
         x::Int
     end
+    # This is written elaborately to defeat inference
+    function hasfoo(list)
+        hf = false
+        hf = map(list) do item
+            if isa(item, AbstractString)
+                (str->occursin("foo", str))(item)
+            else
+                false
+            end
+        end
+        return any(hf)
+    end
+    @generated function Egen(x::T) where T
+        Tbigger = T == Float32 ? Float64 : BigFloat
+        :(convert($Tbigger, x))
+    end
 end

test/runtests.jl

@@ -47,6 +47,16 @@ if VERSION >= v"1.2.0-DEV.573"
         tinf = @snoopi sortperm(rand(5); rev=true)
         pc = SnoopCompile.parcel(tinf)
         @test any(str->occursin("kwftype", str), pc[:Base])
+
+        # Wrap anonymous functions in an `if isdefined`
+        list = Any["xbar7", "yfoo8"]
+        tinf = @snoopi E.hasfoo(list)
+        pc = SnoopCompile.parcel(tinf)
+
+        # Extract the generator in a name-independent manner
+        tinf = @snoopi E.Egen(1.0f0)
+        pc = SnoopCompile.parcel(tinf)
+        @test any(str->occursin("generator", str), pc[:E])
     end
     """)
 
@@ -56,10 +66,11 @@ if VERSION >= v"1.2.0-DEV.573"
 end
 
 # issue #26
-@snoopc "/tmp/anon.log" begin
+logfile = joinpath(tempdir(), "anon.log")
+@snoopc logfile begin
     map(x->x^2, [1,2,3])
 end
-data = SnoopCompile.read("/tmp/anon.log")
+data = SnoopCompile.read(logfile)
 pc = SnoopCompile.parcel(reverse!(data[2]))
 @test length(pc[:Base]) <= 1
 
@@ -69,12 +80,14 @@ keep, pcstring, topmod, name = SnoopCompile.parse_call("Tuple{getfield(JLD, Symb
 @test pcstring == "Tuple{getfield(JLD, Symbol(\"##s27#8\")), Int, Int, Int, Int, Int}"
 @test topmod == :JLD
 @test name == "##s27#8"
-save("/tmp/mat.jld", "mat", sprand(10, 10, 0.1))
-@snoopc "/tmp/jldanon.log" begin
+matfile = joinpath(tempdir(), "mat.jld")
+save(matfile, "mat", sprand(10, 10, 0.1))
+logfile = joinpath(tempdir(), "jldanon.log")
+@snoopc logfile begin
     using JLD, SparseArrays
-    mat = load("/tmp/mat.jld", "mat")
+    mat = load(joinpath(tempdir(), "mat.jld"), "mat")
 end
-data = SnoopCompile.read("/tmp/jldanon.log")
+data = SnoopCompile.read(logfile)
 pc = SnoopCompile.parcel(reverse!(data[2]))
 @test any(startswith.(pc[:JLD], "isdefined"))