Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adapt to Clang 16 internal changes #71

Draft
wants to merge 6 commits into
base: mg/poplar-3.3
Choose a base branch
from
Draft

Adapt to Clang 16 internal changes #71

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
bd9e3ec
Adapt to Clang16's base class name printing policy
Gnimuc Aug 18, 2024
85bfcfb
3.2
Gnimuc Aug 20, 2024
88771b6
3.2 diff
Gnimuc Aug 20, 2024
0270e58
3.3
Gnimuc Aug 20, 2024
f2bfc2a
debug
Gnimuc Aug 20, 2024
4eb3b9a
hacks
Gnimuc Aug 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 66 additions & 15 deletions deps/generate_wrapper.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
##
using Clang
using Clang.LibClang
using Clang.Generators
using JSON
##

Expand Down Expand Up @@ -46,7 +47,7 @@ resolve_headers(headers::Vector{String}, include_paths::Vector{String})::Vector{
resolve_header.(headers, Ref(include_paths))

function get_full_name(cursor, funcargs::Bool=true, buf="")
parent = Clang.getCursorLexicalParent(cursor)
parent = Clang.getCursorSemanticParent(cursor)
parent_kind = spelling(kind(parent))
cursor_name = name(cursor)
if !funcargs
Expand All @@ -66,14 +67,14 @@ end
function get_namespace(cursor::CLCursor)
tmpcursor = cursor

while spelling(kind(tmpcursor)) != "Namespace" && Clang.getCursorLexicalParent(tmpcursor) != tmpcursor
tmpcursor = Clang.getCursorLexicalParent(tmpcursor)
while spelling(kind(tmpcursor)) != "Namespace" && Clang.getCursorSemanticParent(tmpcursor) != tmpcursor
tmpcursor = Clang.getCursorSemanticParent(tmpcursor)
end

if get_full_name(tmpcursor) == ""
tmpcursor = Clang.clang_getCursorDefinition(cursor)
while spelling(kind(tmpcursor)) != "Namespace" && Clang.getCursorLexicalParent(tmpcursor) != tmpcursor
tmpcursor = Clang.getCursorLexicalParent(tmpcursor)
while spelling(kind(tmpcursor)) != "Namespace" && Clang.getCursorSemanticParent(tmpcursor) != tmpcursor
tmpcursor = Clang.getCursorSemanticParent(tmpcursor)
end
return get_full_name(tmpcursor)
end
Expand All @@ -83,8 +84,8 @@ end

function get_class_name(cursor::CLCursor)
tmpcursor = cursor
while spelling(kind(tmpcursor)) != "StructDecl" && spelling(kind(tmpcursor)) != "ClassDecl" && Clang.getCursorLexicalParent(tmpcursor) != tmpcursor
tmpcursor = Clang.getCursorLexicalParent(tmpcursor)
while spelling(kind(tmpcursor)) != "StructDecl" && spelling(kind(tmpcursor)) != "ClassDecl" && Clang.getCursorSemanticParent(tmpcursor) != tmpcursor
tmpcursor = Clang.getCursorSemanticParent(tmpcursor)
end
return get_full_name(tmpcursor)
end
Expand All @@ -111,6 +112,36 @@ function get_julia_name(cursor::CLCursor)
replace(vname, "::" => "")
end

nns(x::Clang.CLInvalidFile) = ""
nns(x::Clang.CLTranslationUnit) = ""
function nns(x::CLCursor)
pn = nns(Clang.getCursorSemanticParent(x))
if isempty(pn)
return spelling(x)
else
return pn * "::" * spelling(x)
end
end
get_qualified_name(x::CLCursor) = isempty(spelling(x)) ? "" : nns(x)

get_qualified_basename(x::CLCursor) = ""
function get_qualified_basename(x::CLTypeRef)
n = spelling(x)
startswith(n, "class ") && return split(n, "class ")[2]
startswith(n, "struct ") && return split(n, "struct ")[2]
startswith(n, "Union ") && return split(n, "Union ")[2]
return ""
end
get_qualified_basename(x::CLCXXBaseSpecifier) = get_qualified_basename(children(x)[1])
function get_qualified_basename(x::Union{CLClassDecl,CLStructDecl,CLUnionDecl})
bn = get_qualified_basename(children(x)[1])
if isempty(bn)
return spelling(x)
else
return bn * "::" * spelling(x)
end
end

function object_decl_handler(ctx::BindgenContext, classdecl::CLCursor)::Tuple{Union{Nothing, String}, Union{Nothing, String}}
full_name = get_full_name(classdecl)
length(children(classdecl)) == 0 && return nothing, "skip_empty_classdecl"
Expand All @@ -123,10 +154,10 @@ function object_decl_handler(ctx::BindgenContext, classdecl::CLCursor)::Tuple{Un
end

# handle simple inheritance
if length(children(classdecl)) > 1 && kind(children(classdecl)[1]) == Clang.CXCursor_CXXBaseSpecifier

if startswith(get_full_name(children(classdecl)[1]), "class ")
base_class = split(get_full_name(children(classdecl)[1]), "class ")[2]
subnodes = children(classdecl)
if length(subnodes) > 1 && kind(subnodes[1]) == Clang.CXCursor_CXXBaseSpecifier
base_class = get_qualified_basename(subnodes[1])
if !isempty(base_class)

ctx.outputSupertypes *= "template<> struct SuperType<$full_name> { typedef $base_class type; };\n"

Expand Down Expand Up @@ -158,7 +189,7 @@ end
function arg_list(method::CLCursor, types=true::Bool, cutoff=Inf, varnames=true::Bool)
Clang.getNumArguments(Clang.getCursorType(method)) == 0 && return ""
cutoff == 0 && return ""
arglist = get_full_name(method)
arglist = get_full_name(method) # -> spelling(method)
arglist = split(arglist, "(")[2]
arglist = split(arglist, ")")[1]
# TODO: this is **really** not a good way to do this
Expand Down Expand Up @@ -200,6 +231,10 @@ function arg_list(method::CLCursor, types=true::Bool, cutoff=Inf, varnames=true:
item = replace(item, "poplar::DebugContext" => "std::string")
item = replace(item, "poplar::StringRef" => "std::string")

# item = replace(item, "const DebugContext &" => "std::string")
# item = replace(item, "DebugContext" => "std::string")
# item = replace(item, "StringRef" => "std::string")

if types
pre = ""
if should_wrap(item)
Expand Down Expand Up @@ -389,7 +424,7 @@ function enum_const_handler(ctx::BindgenContext, decl::CLCursor)::Tuple{Union{No

full_name = get_full_name(decl)
julia_name = get_julia_name(decl)
parent_name = get_julia_name(Clang.getCursorLexicalParent(decl))
parent_name = get_julia_name(Clang.getCursorSemanticParent(decl))
return "mod.set_const(\"$parent_name$julia_name\", $full_name);", nothing
end

Expand Down Expand Up @@ -436,6 +471,7 @@ function iterate_children(ctx::BindgenContext, childvec::Vector{CLCursor})

child_id = get_full_name(child) * "__" * spelling(child_kind)
child_id = replace(child_id, "poplar::StringRef" => "std::string")
# child_id = replace(child_id, "StringRef" => "std::string")

# prevents duplicate codegen(+error), TODO: check if still necessary
child_id == "poplar::FieldData::SizeT::size()__CXXMethod" && (valid = false; reason = "filedata_size_blacklist")
Expand All @@ -461,29 +497,35 @@ function iterate_children(ctx::BindgenContext, childvec::Vector{CLCursor})

# This conversion `ArrayRef<std::string>` to `ArrayRef<poplar::StringRef>` isn't handled correctly
contains(child_id, "poplar::Graph::trace(ArrayRef<std::string>") && (valid = false; reason = "arrayrefstring_blacklisted")
# contains(child_id, "trace(ArrayRef<std::string>") && (valid = false; reason = "arrayrefstring_blacklisted")

# `DebugContext` constructors which cause ambiguous overload calls
contains(child_id, r"^poplar::DebugContext::DebugContext.*__CXXConstructor$") && (valid = false; reason = "debugcontext_blacklisted")
# contains(child_id, r"^DebugContext.*__CXXConstructor$") && (valid = false; reason = "debugcontext_blacklisted")

# This causes the error
# no matching function for call to ‘poplar::program::Sequence::add_many(std::__cxx11::basic_string<char>&)’
contains(child_id, r"poplar::program::Sequence::Sequence.*__CXXConstructor$") && (valid = false; reason = "programsequence_blacklisted")

# Avoid duplicate definition during precompilation of the CxxWrap module
contains(child_id, "poplar::layout::to_string(const poplar::layout::VectorList)__FunctionDecl") && (valid = false; reason = "duplicate_definition")
# contains(child_id, "poplar::layout::to_string(const VectorList)__FunctionDecl") && (valid = false; reason = "duplicate_definition")

# Avoid duplicate definition during precompilation of the CxxWrap module.
# Ref: <https://github.com/JuliaIPU/IPUToolkit.jl/issues/12>.
contains(child_id, "poplar::toString") && (valid = false; reason = "duplicate_definition")

# error: invalid use of incomplete type ‘class pva::Report’
contains(child_id, "poplar::Engine::getReport") && (valid = false; reason = "incomplete_type")
# contains(child_id, "Engine::getReport") && (valid = false; reason = "incomplete_type")

# error: invalid application of ‘sizeof’ to incomplete type ‘poplar::core::VertexIntrospector’
contains(child_id, "poplar::VertexIntrospector") && (valid = false; reason = "incomplete_type")
# contains(child_id, "VertexIntrospector") && (valid = false; reason = "incomplete_type")

# error: invalid use of incomplete type ‘class poplar::Preallocations’
contains(child_id, "poplar::Preallocations") && (valid = false; reason = "incomplete_type")
# contains(child_id, "Preallocations") && (valid = false; reason = "incomplete_type")

# error: no matching function for call to ‘poplar::GlobalExchangeConstraint::GlobalExchangeConstraint()’
contains(child_id, "poplar::Target::getGlobalExchangeConstraints()__CXXMethod") && (valid = false; reason = "getGlobalExchangeConstraints_blacklisted")
Expand Down Expand Up @@ -576,7 +618,7 @@ function gen_bindings(headers::Vector{String}, blacklist::Vector{String})
println(io, "#include <$(header)>")
end

includes = get_system_includes()
includes = [joinpath(@__DIR__, "wrapper", "3.3", "include")]
ctx = DefaultBindgenContext()
ctx.searched_headers = resolve_headers(headers, includes)
ctx.blacklisted_headers = resolve_headers(blacklist, includes)
Expand All @@ -588,7 +630,14 @@ function gen_bindings(headers::Vector{String}, blacklist::Vector{String})
tus = []
symbol_names = String[]
# add compiler flags
clang_args = ["-I"*inc for inc in includes]
# clang_args = ["-I"*inc for inc in includes]
# cross-compiling setup for DEBUG:
args = get_default_args("x86_64-linux-gnu", is_cxx=true, version=v"9.1.0")
append!(args, ["-I"*inc for inc in includes])
push!(args, "-nostdinc++", "-nostdlib++")
push!(args, "-xc++")
# push!(args, "-v")
clang_args = args
for h in ctx.searched_headers
tu = Clang.TranslationUnit(idx, h, clang_args, flags)
push!(tus, tu)
Expand Down Expand Up @@ -629,3 +678,5 @@ function generate_wrapper()

return nothing
end

generate_wrapper()
8 changes: 8 additions & 0 deletions deps/wrapper/gen_headers.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#include <poplar/VectorLayout.hpp>
#include <poplar/DeviceManager.hpp>
#include <poplar/Engine.hpp>
#include <poplar/Graph.hpp>
#include <poplar/CSRFunctions.hpp>
#include <poplar/IPUModel.hpp>
#include <popops/ElementWise.hpp>
#include <popops/codelets.hpp>
62 changes: 62 additions & 0 deletions deps/wrapper/gen_inherit.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
template<> struct SuperType<poplar::graph_program_compilation_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::graph_object_creation_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::graph_object_load_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::no_environment> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::unknown_vertex_type> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::unknown_field> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::control_program_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::type_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::index_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::no_size_specified> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::graph_connection_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::graph_cycle_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::graph_recursion_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::graph_replication_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::profiling_disabled> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::missing_profile_data> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::missing_graph_profile> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::missing_perf_estimate> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::invalid_tile_mapping> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::tensor_creation_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::tensor_io_state_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::tensor_metadata_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::stream_connection_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::overflow_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::memory_elem_constraints_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::graph_memory_allocation_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::stream_memory_allocation_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::invalid_machine_model> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::invalid_option> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::link_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::runtime_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::application_runtime_error> { typedef poplar::runtime_error type; };
template<> struct SuperType<poplar::system_runtime_error> { typedef poplar::runtime_error type; };
template<> struct SuperType<poplar::recoverable_runtime_error> { typedef poplar::system_runtime_error type; };
template<> struct SuperType<poplar::unrecoverable_runtime_error> { typedef poplar::system_runtime_error type; };
template<> struct SuperType<poplar::unknown_runtime_error> { typedef poplar::system_runtime_error type; };
template<> struct SuperType<poplar::symbol_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::file_load_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::parse_error> { typedef poplar::poplar_error type; };
template<> struct SuperType<poplar::program::Execute> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::Sequence> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::Repeat> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::RepeatWhileFalse> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::RepeatWhileTrue> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::Loop> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::If> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::Switch> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::Copy> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::CrossReplicaCopy> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::WriteUndef> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::AssumeEqualAcrossReplicas> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::Block> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::Sync> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::Call> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::PrintTensor> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::ErrorProgram> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::Abort> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::program::AbortOnCondition> { typedef poplar::program::Program type; };
template<> struct SuperType<poplar::StreamCallback> { typedef poplar::StreamCallbackBase type; };
template<> struct SuperType<poplar::StreamCallbackWithMetadata> { typedef poplar::StreamCallback type; };
template<> struct SuperType<poplar::ResumableStreamCallback> { typedef poplar::StreamCallback type; };
template<> struct SuperType<poplar::LegacyStreamCallback> { typedef poplar::StreamCallback type; };
Loading
Loading