diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e97574f..7157843 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,30 +10,31 @@ jobs: strategy: matrix: - otp: ['25', '26', '27'] - rebar: ['3.24'] + otp: ['26', '27', '28'] + rebar: ['3.25'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: erlef/setup-beam@v1 id: setup-beam with: otp-version: ${{matrix.otp}} rebar3-version: ${{matrix.rebar}} - name: Restore _build - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: _build key: _build-cache-for-os-${{runner.os}}-otp-${{steps.setup-beam.outputs.otp-version}}-rebar3-${{steps.setup-beam.outputs.rebar3-version}}-hash-${{hashFiles('rebar.lock')}} - name: Restore rebar3's cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/rebar3 key: rebar3-cache-for-os-${{runner.os}}-otp-${{steps.setup-beam.outputs.otp-version}}-rebar3-${{steps.setup-beam.outputs.rebar3-version}}-hash-${{hashFiles('rebar.lock')}} - name: Compile run: ERL_FLAGS="-enable-feature all" rebar3 compile - #- name: Format check - # run: ERL_FLAGS="-enable-feature all" rebar3 format --verify + - name: Format check + if: matrix.otp == '27' + run: rebar3 fmt --check - name: Run tests and verifications (features not enabled) run: rebar3 test - name: Run tests and verifications (features enabled) diff --git a/elvis.config b/elvis.config index 7f97b4f..0f84675 100644 --- a/elvis.config +++ b/elvis.config @@ -1,10 +1,20 @@ -[{elvis, - [{config, - [#{dirs => ["src"], - filter => "*.erl", - ruleset => erl_files, - rules => [{elvis_style, atom_naming_convention, #{regex => "^([a-z][A-Za-z0-9]*_?)*$"}}]}, - #{dirs => ["test"], - filter => "*.erl", - ruleset => erl_files, - rules => [{elvis_style, no_debug_call, disable}]}]}]}]. +[ + {elvis, [ + {config, [ + #{ + dirs => ["src"], + filter => "*.erl", + ruleset => erl_files, + rules => [ + {elvis_style, atom_naming_convention, #{regex => "^([a-z][A-Za-z0-9]*_?)*$"}} + ] + }, + #{ + dirs => ["test"], + filter => "*.erl", + ruleset => erl_files, + rules => [{elvis_style, no_debug_call, disable}] + } + ]} + ]} +]. diff --git a/rebar.config b/rebar.config index 5d0d0df..38624e4 100644 --- a/rebar.config +++ b/rebar.config @@ -1,45 +1,48 @@ %% == Compiler and Profiles == -{erl_opts, - [warn_unused_import, warn_export_vars, warnings_as_errors, verbose, report, debug_info]}. +{erl_opts, [warn_unused_import, warn_export_vars, warnings_as_errors, verbose, report, debug_info]}. -{minimum_otp_vsn, "25"}. +{minimum_otp_vsn, "26"}. -{profiles, - [{test, [{cover_enabled, true}, {cover_opts, [verbose]}, {ct_opts, [{verbose, true}]}]}]}. +{profiles, [{test, [{cover_enabled, true}, {cover_opts, [verbose]}, {ct_opts, [{verbose, true}]}]}]}. {alias, [{test, [compile, lint, xref, dialyzer, ct, cover]}]}. %% == Dependencies and plugins == -{project_plugins, - [{rebar3_hank, "~> 1.4.0"}, - {rebar3_hex, "~> 7.0.7"}, - {rebar3_format, "~> 1.3.0"}, - {rebar3_lint, "~> 3.1.0"}, - {rebar3_ex_doc, "~> 0.2.20"}]}. +{project_plugins, [ + {rebar3_hank, "~> 1.4.0"}, + {rebar3_hex, "~> 7.0.7"}, + {erlfmt, "~> 1.6.2"}, + {rebar3_lint, "~> 3.1.0"}, + {rebar3_ex_doc, "~> 0.2.20"} +]}. %% == Documentation == -{ex_doc, - [{source_url, <<"https://github.com/inaka/katana-code">>}, - {extras, [<<"README.md">>, <<"LICENSE">>]}, - {main, <<"README.md">>}, - {prefix_ref_vsn_with_v, false}]}. +{ex_doc, [ + {source_url, <<"https://github.com/inaka/katana-code">>}, + {extras, [<<"README.md">>, <<"LICENSE">>]}, + {main, <<"README.md">>}, + {prefix_ref_vsn_with_v, false} +]}. {hex, [{doc, #{provider => ex_doc}}]}. %% == Format == -{format, [{options, #{unquote_atoms => false}}]}. +{erlfmt, [ + write, + {files, ["src/**/*.app.src", "src/**/*.erl", "test/**/*.erl", "*.config"]} +]}. %% == Dialyzer + XRef == -{dialyzer, - [{warnings, [no_return, unmatched_returns, error_handling, underspecs, unknown]}, - {plt_extra_apps, [syntax_tools, common_test]}]}. +{dialyzer, [ + {warnings, [no_return, unmatched_returns, error_handling, underspecs, unknown]}, + {plt_extra_apps, [syntax_tools, common_test]} +]}. -{xref_checks, - [undefined_function_calls, deprecated_function_calls, deprecated_functions]}. +{xref_checks, [undefined_function_calls, deprecated_function_calls, deprecated_functions]}. {xref_extra_paths, ["test/**"]}. diff --git a/src/katana_code.app.src b/src/katana_code.app.src index afb6d49..f645cf0 100644 --- a/src/katana_code.app.src +++ b/src/katana_code.app.src @@ -1,10 +1,10 @@ -{application, - katana_code, - [{description, "Functions useful for processing Erlang code."}, - {vsn, git}, - {applications, [kernel, stdlib]}, - {modules, []}, - {registered, []}, - {licenses, ["Apache 2.0"]}, - {links, [{"GitHub", "https://github.com/inaka/katana-code"}]}, - {build_tools, ["rebar3"]}]}. +{application, katana_code, [ + {description, "Functions useful for processing Erlang code."}, + {vsn, git}, + {applications, [kernel, stdlib]}, + {modules, []}, + {registered, []}, + {licenses, ["Apache 2.0"]}, + {links, [{"GitHub", "https://github.com/inaka/katana-code"}]}, + {build_tools, ["rebar3"]} +]}. diff --git a/src/ktn_code.erl b/src/ktn_code.erl index 572f4d6..9bd1fd1 100644 --- a/src/ktn_code.erl +++ b/src/ktn_code.erl @@ -13,23 +13,14 @@ -export_type([tree_node/0, tree_node_type/0, beam_lib_beam/0]). %% NOTE: we use atom() below, because erl_scan:category() is not exported. -%% In fact, this type ends up being just atom() for dialyzer, -%% since it has too many options and it's compressed. --type tree_node_type() :: - 'case' | 'catch' | 'else' | 'fun' | 'if' | 'maybe' | 'receive' | 'try' | any | atom | - b_generate | bc | bc_expr | binary | binary_element | block | call | callback | - case_clauses | case_expr | char | clause | comment | cons | default | define | else_attr | - export | float | function | generate | if_attr | import | integer | lc | lc_expr | - m_generate | macro | map | map_field_assoc | map_field_exact | match | maybe_match | mc | - mc_expr | module | named_fun | nil | nominal | op | opaque | query | receive_after | - receive_case | record | record_attr | record_field | record_index | remote | remote_type | - root | spec | string | try_after | try_case | try_catch | tuple | type | type_attr | - type_map_field | typed_record_field | user_type | var | atom(). +-type tree_node_type() :: atom(). -type tree_node() :: - #{type => tree_node_type(), - attrs => map(), - node_attrs => map(), - content => [tree_node()]}. + #{ + type => tree_node_type(), + attrs => map(), + node_attrs => map(), + content => [tree_node()] + }. -type beam_lib_beam() :: file:filename() | binary(). % Should eventually become beam_lib:beam(), once that's exposed % (https://github.com/erlang/otp/pull/7534) @@ -37,15 +28,11 @@ % Should eventually become erl_syntax:annotation_or_location(), once that's exposed % (https://github.com/erlang/otp/pull/7535) -type erl_parse_foo() :: - {attribute, - Pos :: erl_syntax_annotation_or_location(), - Name :: erl_syntax:syntaxTree(), - Args :: none | [erl_syntax:syntaxTree()]} | - {macro, - Pos :: erl_syntax_annotation_or_location(), - Name :: erl_syntax:syntaxTree(), - Args :: none | [erl_syntax:syntaxTree()]} | - {atom, [{node, Node :: erl_syntax:syntaxTree()}], non_reversible_form}. + {attribute, Pos :: erl_syntax_annotation_or_location(), Name :: erl_syntax:syntaxTree(), + Args :: none | [erl_syntax:syntaxTree()]} + | {macro, Pos :: erl_syntax_annotation_or_location(), Name :: erl_syntax:syntaxTree(), + Args :: none | [erl_syntax:syntaxTree()]} + | {atom, [{node, Node :: erl_syntax:syntaxTree()}], non_reversible_form}. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Exported API @@ -59,7 +46,8 @@ beam_to_string(BeamPath) -> case beam_lib:chunks(BeamPath, [abstract_code]) of {ok, {_, [{abstract_code, {raw_abstract_v1, Forms}}]}} -> Src = erl_prettypr:format( - erl_syntax:form_list(tl(Forms))), + erl_syntax:form_list(tl(Forms)) + ), {ok, Src}; Error -> Error @@ -92,14 +80,18 @@ parse_tree(Source) -> Comments = lists:filter(fun is_comment/1, Tokens), Children = - [to_map(Form) + [ + to_map(Form) || Form <- Forms, %% filter forms that couldn't be parsed - element(1, Form) =/= error], + element(1, Form) =/= error + ], - #{type => root, - attrs => #{tokens => lists:map(fun token_to_map/1, Tokens)}, - content => to_map(Comments) ++ Children}. + #{ + type => root, + attrs => #{tokens => lists:map(fun token_to_map/1, Tokens)}, + content => to_map(Comments) ++ Children + }. -spec is_comment(erl_scan:token()) -> boolean(). is_comment({comment, _, _}) -> @@ -130,8 +122,10 @@ revert(attribute, Node0) -> Node = erl_syntax:update_tree(Node0, Gs), Name = - try erl_syntax:atom_value( - erl_syntax:attribute_name(Node)) + try + erl_syntax:atom_value( + erl_syntax:attribute_name(Node) + ) of 'if' -> if_attr; @@ -188,14 +182,14 @@ consult(Source) -> Forms = split_when(fun is_dot/1, Tokens), ParseFun = fun(Form) -> - {ok, Expr} = erl_parse:parse_exprs(Form), - Expr + {ok, Expr} = erl_parse:parse_exprs(Form), + Expr end, Parsed = lists:map(ParseFun, Forms), ExprsFun = fun(P) -> - {value, Value, _} = erl_eval:exprs(P, []), - Value + {value, Value, _} = erl_eval:exprs(P, []), + Value end, lists:map(ExprsFun, Parsed). @@ -308,397 +302,601 @@ get_text(_Attrs) -> to_map(ListParsed) when is_list(ListParsed) -> lists:map(fun to_map/1, ListParsed); to_map({function, Attrs, Name, Arity, Clauses}) -> - #{type => function, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - name => Name, - arity => Arity}, - content => to_map(Clauses)}; + #{ + type => function, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + name => Name, + arity => Arity + }, + content => to_map(Clauses) + }; to_map({function, Name, Arity}) -> #{type => function, attrs => #{name => Name, arity => Arity}}; to_map({function, Module, Name, Arity}) -> - #{type => function, - attrs => - #{module => Module, - name => Name, - arity => Arity}}; + #{ + type => function, + attrs => + #{ + module => Module, + name => Name, + arity => Arity + } + }; to_map({clause, Attrs, Patterns, Guards, Body}) -> - #{type => clause, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - node_attrs => #{pattern => to_map(Patterns), guards => to_map(Guards)}, - content => to_map(Body)}; + #{ + type => clause, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => #{pattern => to_map(Patterns), guards => to_map(Guards)}, + content => to_map(Body) + }; to_map({match, Attrs, Left, Right}) -> - #{type => match, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => to_map([Left, Right])}; + #{ + type => match, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => to_map([Left, Right]) + }; to_map({maybe_match, Attrs, Left, Right}) -> - #{type => maybe_match, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => to_map([Left, Right])}; + #{ + type => maybe_match, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => to_map([Left, Right]) + }; to_map({tuple, Attrs, Elements}) -> - #{type => tuple, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => to_map(Elements)}; + #{ + type => tuple, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => to_map(Elements) + }; %% Literals -to_map({Type, Attrs, Value}) - when Type == atom; Type == integer; Type == float; Type == string; Type == char -> - #{type => Type, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - value => Value}}; +to_map({Type, Attrs, Value}) when + Type == atom; Type == integer; Type == float; Type == string; Type == char +-> + #{ + type => Type, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + value => Value + } + }; to_map({bin, Attrs, Elements}) -> - #{type => binary, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => to_map(Elements)}; + #{ + type => binary, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => to_map(Elements) + }; to_map({bin_element, Attrs, Value, Size, TSL}) -> - #{type => binary_element, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - type_spec_list => TSL}, - node_attrs => - #{value => to_map(Value), - size => - case Size of - default -> - #{type => default}; - _ -> - to_map(Size) - end}}; + #{ + type => binary_element, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + type_spec_list => TSL + }, + node_attrs => + #{ + value => to_map(Value), + size => + case Size of + default -> + #{type => default}; + _ -> + to_map(Size) + end + } + }; %% Variables to_map({var, Attrs, Name}) -> - #{type => var, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - name => Name}}; + #{ + type => var, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + name => Name + } + }; %% Function call to_map({call, Attrs, Function, Arguments}) -> - #{type => call, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - node_attrs => #{function => to_map(Function)}, - content => to_map(Arguments)}; + #{ + type => call, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => #{function => to_map(Function)}, + content => to_map(Arguments) + }; to_map({remote, Attrs, Module, Function}) -> - #{type => remote, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - node_attrs => #{module => to_map(Module), function => to_map(Function)}}; + #{ + type => remote, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => #{module => to_map(Module), function => to_map(Function)} + }; %% case to_map({'case', Attrs, Expr, Clauses}) -> CaseExpr = to_map({case_expr, Attrs, Expr}), CaseClauses = to_map({case_clauses, Attrs, Clauses}), - #{type => 'case', - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - node_attrs => #{expression => to_map(Expr)}, - content => [CaseExpr, CaseClauses]}; + #{ + type => 'case', + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => #{expression => to_map(Expr)}, + content => [CaseExpr, CaseClauses] + }; to_map({case_expr, Attrs, Expr}) -> - #{type => case_expr, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => [to_map(Expr)]}; + #{ + type => case_expr, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => [to_map(Expr)] + }; to_map({case_clauses, Attrs, Clauses}) -> - #{type => case_clauses, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => to_map(Clauses)}; + #{ + type => case_clauses, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => to_map(Clauses) + }; %% fun to_map({'fun', Attrs, {function, Name, Arity}}) -> - #{type => 'fun', - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - name => Name, - arity => Arity}}; + #{ + type => 'fun', + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + name => Name, + arity => Arity + } + }; to_map({'fun', Attrs, {function, Module, Name, Arity}}) -> - #{type => 'fun', - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - module => Module, - name => Name, - arity => Arity}}; + #{ + type => 'fun', + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + module => Module, + name => Name, + arity => Arity + } + }; to_map({'fun', Attrs, {clauses, Clauses}}) -> - #{type => 'fun', - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => to_map(Clauses)}; + #{ + type => 'fun', + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => to_map(Clauses) + }; to_map({named_fun, Attrs, Name, Clauses}) -> - #{type => named_fun, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - name => Name}, - content => to_map(Clauses)}; + #{ + type => named_fun, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + name => Name + }, + content => to_map(Clauses) + }; %% query - deprecated, implemented for completion. to_map({query, Attrs, ListCompr}) -> - #{type => query, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => to_map(ListCompr)}; + #{ + type => query, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => to_map(ListCompr) + }; %% try..catch..after to_map({'try', Attrs, Body, [], CatchClauses, AfterBody}) -> TryBody = to_map(Body), TryCatch = to_map({try_catch, Attrs, CatchClauses}), TryAfter = to_map({try_after, Attrs, AfterBody}), - #{type => 'try', - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - node_attrs => #{catch_clauses => to_map(CatchClauses), after_body => to_map(AfterBody)}, - content => TryBody ++ [TryCatch, TryAfter]}; + #{ + type => 'try', + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => #{catch_clauses => to_map(CatchClauses), after_body => to_map(AfterBody)}, + content => TryBody ++ [TryCatch, TryAfter] + }; %% try..of..catch..after to_map({'try', Attrs, Expr, CaseClauses, CatchClauses, AfterBody}) -> TryCase = to_map({try_case, Attrs, Expr, CaseClauses}), TryCatch = to_map({try_catch, Attrs, CatchClauses}), TryAfter = to_map({try_after, Attrs, AfterBody}), - #{type => 'try', - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => [TryCase, TryCatch, TryAfter]}; + #{ + type => 'try', + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => [TryCase, TryCatch, TryAfter] + }; to_map({try_case, Attrs, Expr, Clauses}) -> - #{type => try_case, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - node_attrs => #{expression => to_map(Expr)}, - content => to_map(Clauses)}; + #{ + type => try_case, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => #{expression => to_map(Expr)}, + content => to_map(Clauses) + }; to_map({try_catch, Attrs, Clauses}) -> - #{type => try_catch, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => to_map(Clauses)}; + #{ + type => try_catch, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => to_map(Clauses) + }; to_map({try_after, Attrs, AfterBody}) -> - #{type => try_after, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => to_map(AfterBody)}; + #{ + type => try_after, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => to_map(AfterBody) + }; %% maybe..end to_map({'maybe', Attrs, Body}) -> MaybeBody = to_map(Body), - #{type => 'maybe', - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => MaybeBody}; + #{ + type => 'maybe', + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => MaybeBody + }; %% maybe..else..end to_map({'maybe', Attrs, Body, Else}) -> MaybeBody = to_map(Body), MaybeElse = to_map(Else), - #{type => 'maybe', - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => MaybeBody ++ [MaybeElse]}; + #{ + type => 'maybe', + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => MaybeBody ++ [MaybeElse] + }; to_map({'else', Attrs, Clauses}) -> - #{type => 'else', - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => to_map(Clauses)}; + #{ + type => 'else', + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => to_map(Clauses) + }; %% if to_map({'if', Attrs, IfClauses}) -> - #{type => 'if', - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => to_map(IfClauses)}; + #{ + type => 'if', + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => to_map(IfClauses) + }; %% catch to_map({'catch', Attrs, Expr}) -> - #{type => 'catch', - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => [to_map(Expr)]}; + #{ + type => 'catch', + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => [to_map(Expr)] + }; %% receive to_map({'receive', Attrs, Clauses}) -> RecClauses = to_map({receive_case, Attrs, Clauses}), - #{type => 'receive', - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => [RecClauses]}; + #{ + type => 'receive', + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => [RecClauses] + }; to_map({'receive', Attrs, Clauses, AfterExpr, AfterBody}) -> RecClauses = to_map({receive_case, Attrs, Clauses}), RecAfter = to_map({receive_after, Attrs, AfterExpr, AfterBody}), - #{type => 'receive', - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => [RecClauses, RecAfter]}; + #{ + type => 'receive', + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => [RecClauses, RecAfter] + }; to_map({receive_case, Attrs, Clauses}) -> - #{type => receive_case, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => to_map(Clauses)}; + #{ + type => receive_case, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => to_map(Clauses) + }; to_map({receive_after, Attrs, Expr, Body}) -> - #{type => receive_after, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - node_attrs => #{expression => to_map(Expr)}, - content => to_map(Body)}; + #{ + type => receive_after, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => #{expression => to_map(Expr)}, + content => to_map(Body) + }; %% List to_map({nil, Attrs}) -> #{type => nil, attrs => #{location => get_location(Attrs), text => get_text(Attrs)}}; to_map({cons, Attrs, Head, Tail}) -> - #{type => cons, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => [to_map(Head), to_map(Tail)]}; + #{ + type => cons, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => [to_map(Head), to_map(Tail)] + }; %% Map to_map({map, Attrs, Pairs}) -> - #{type => map, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => to_map(Pairs)}; + #{ + type => map, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => to_map(Pairs) + }; to_map({map, Attrs, Var, Pairs}) -> - #{type => map, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - node_attrs => #{var => to_map(Var)}, - content => to_map(Pairs)}; + #{ + type => map, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => #{var => to_map(Var)}, + content => to_map(Pairs) + }; to_map({Type, Attrs, Key, Value}) when map_field_exact == Type; map_field_assoc == Type -> - #{type => Type, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - node_attrs => #{key => to_map(Key), value => to_map(Value)}}; + #{ + type => Type, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => #{key => to_map(Key), value => to_map(Value)} + }; %% List Comprehension to_map({lc, Attrs, Expr, GeneratorsFilters}) -> LcExpr = to_map({lc_expr, Attrs, Expr}), LcGenerators = to_map(GeneratorsFilters), - #{type => lc, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => [LcExpr | LcGenerators]}; + #{ + type => lc, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => [LcExpr | LcGenerators] + }; to_map({generate, Attrs, Pattern, Expr}) -> - #{type => generate, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - node_attrs => #{pattern => to_map(Pattern), expression => to_map(Expr)}}; + #{ + type => generate, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => #{pattern => to_map(Pattern), expression => to_map(Expr)} + }; +to_map({zip, Attrs, Generators}) -> + #{ + type => zip, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + generators => [to_map(Generator) || Generator <- Generators] + } + }; +to_map({generate_strict, Attrs, Pattern, Expr}) -> + #{ + type => generate_strict, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => #{pattern => to_map(Pattern), expression => to_map(Expr)} + }; to_map({lc_expr, Attrs, Expr}) -> - #{type => lc_expr, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => [to_map(Expr)]}; + #{ + type => lc_expr, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => [to_map(Expr)] + }; %% Binary Comprehension to_map({bc, Attrs, Expr, GeneratorsFilters}) -> BcExpr = to_map({bc_expr, Attrs, Expr}), BcGenerators = to_map(GeneratorsFilters), - #{type => bc, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => [BcExpr | BcGenerators]}; + #{ + type => bc, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => [BcExpr | BcGenerators] + }; to_map({b_generate, Attrs, Pattern, Expr}) -> - #{type => b_generate, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - node_attrs => #{pattern => to_map(Pattern), expression => to_map(Expr)}}; + #{ + type => b_generate, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => #{pattern => to_map(Pattern), expression => to_map(Expr)} + }; +to_map({b_generate_strict, Attrs, Pattern, Expr}) -> + #{ + type => b_generate_strict, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => #{pattern => to_map(Pattern), expression => to_map(Expr)} + }; to_map({bc_expr, Attrs, Expr}) -> - #{type => bc_expr, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => [to_map(Expr)]}; + #{ + type => bc_expr, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => [to_map(Expr)] + }; %% Map Comprehension to_map({mc, Anno, RepE0, RepQs}) -> McExpr = to_map({mc_expr, Anno, RepE0}), McGenerators = to_map(RepQs), - #{type => mc, - attrs => #{location => get_location(Anno), text => get_text(Anno)}, - content => [McExpr | McGenerators]}; + #{ + type => mc, + attrs => #{location => get_location(Anno), text => get_text(Anno)}, + content => [McExpr | McGenerators] + }; to_map({m_generate, Anno, Pattern, RepE0}) -> - #{type => m_generate, - attrs => #{location => get_location(Anno), text => get_text(Anno)}, - node_attrs => #{pattern => to_map(Pattern), expression => to_map(RepE0)}}; + #{ + type => m_generate, + attrs => #{location => get_location(Anno), text => get_text(Anno)}, + node_attrs => #{pattern => to_map(Pattern), expression => to_map(RepE0)} + }; +to_map({m_generate_strict, Anno, Pattern, RepE0}) -> + #{ + type => m_generate_strict, + attrs => #{location => get_location(Anno), text => get_text(Anno)}, + node_attrs => #{pattern => to_map(Pattern), expression => to_map(RepE0)} + }; to_map({mc_expr, Anno, RepE0}) -> - #{type => mc_expr, - attrs => #{location => get_location(Anno), text => get_text(Anno)}, - content => [to_map(RepE0)]}; + #{ + type => mc_expr, + attrs => #{location => get_location(Anno), text => get_text(Anno)}, + content => [to_map(RepE0)] + }; %% Operation to_map({op, Attrs, Operation, Left, Right}) -> - #{type => op, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - operation => Operation}, - content => to_map([Left, Right])}; + #{ + type => op, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + operation => Operation + }, + content => to_map([Left, Right]) + }; to_map({op, Attrs, Operation, Single}) -> - #{type => op, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - operation => Operation}, - content => to_map([Single])}; + #{ + type => op, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + operation => Operation + }, + content => to_map([Single]) + }; %% Record to_map({record, Attrs, Name, Fields}) -> - #{type => record, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - name => Name}, - content => to_map(Fields)}; + #{ + type => record, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + name => Name + }, + content => to_map(Fields) + }; to_map({record, Attrs, Var, Name, Fields}) -> - #{type => record, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - name => Name}, - node_attrs => #{variable => to_map(Var)}, - content => to_map(Fields)}; + #{ + type => record, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + name => Name + }, + node_attrs => #{variable => to_map(Var)}, + content => to_map(Fields) + }; to_map({record_index, Attrs, Name, Field}) -> - #{type => record_index, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - name => Name}, - content => [to_map(Field)]}; + #{ + type => record_index, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + name => Name + }, + content => [to_map(Field)] + }; to_map({record_field, Attrs, Name}) -> - #{type => record_field, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - node_attrs => #{name => to_map(Name)}}; + #{ + type => record_field, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => #{name => to_map(Name)} + }; to_map({record_field, Attrs, Name, Default}) -> - #{type => record_field, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - node_attrs => #{default => to_map(Default), name => to_map(Name)}}; + #{ + type => record_field, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => #{default => to_map(Default), name => to_map(Name)} + }; to_map({record_field, Attrs, Var, Name, Field}) -> - #{type => record_field, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - name => Name}, - node_attrs => #{variable => to_map(Var)}, - content => [to_map(Field)]}; + #{ + type => record_field, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + name => Name + }, + node_attrs => #{variable => to_map(Var)}, + content => [to_map(Field)] + }; %% Block to_map({block, Attrs, Body}) -> - #{type => block, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - content => to_map(Body)}; + #{ + type => block, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + content => to_map(Body) + }; %% Record Attribute to_map({attribute, Attrs, record, {Name, Fields}}) -> - #{type => record_attr, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - name => Name}, - content => to_map(Fields)}; + #{ + type => record_attr, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + name => Name + }, + content => to_map(Fields) + }; to_map({typed_record_field, Field, Type}) -> FieldMap = to_map(Field), - #{type => typed_record_field, - attrs => - #{location => attr(location, FieldMap), - text => attr(text, FieldMap), - field => FieldMap}, - node_attrs => #{type => to_map(Type)}}; + #{ + type => typed_record_field, + attrs => + #{ + location => attr(location, FieldMap), + text => attr(text, FieldMap), + field => FieldMap + }, + node_attrs => #{type => to_map(Type)} + }; %% Type to_map({type, Attrs, 'fun', Types}) -> - #{type => type, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - name => 'fun'}, - content => to_map(Types)}; + #{ + type => type, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + name => 'fun' + }, + content => to_map(Types) + }; to_map({type, Attrs, constraint, [Sub, SubType]}) -> - #{type => type, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - name => constraint, - subtype => Sub}, - content => to_map(SubType)}; + #{ + type => type, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + name => constraint, + subtype => Sub + }, + content => to_map(SubType) + }; to_map({type, Attrs, bounded_fun, [FunType, Defs]}) -> - #{type => type, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - name => bounded_fun}, - node_attrs => #{'fun' => to_map(FunType)}, - content => to_map(Defs)}; + #{ + type => type, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + name => bounded_fun + }, + node_attrs => #{'fun' => to_map(FunType)}, + content => to_map(Defs) + }; to_map({type, Attrs, Name, any}) -> to_map({type, Attrs, Name, [any]}); to_map({type, Attrs, any}) -> - #{type => type, - attrs => - #{location => get_location(Attrs), - text => "...", - name => '...'}}; + #{ + type => type, + attrs => + #{ + location => get_location(Attrs), + text => "...", + name => '...' + } + }; to_map({type, Attrs, Name, Types}) -> - #{type => type, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - name => Name}, - content => to_map(Types)}; -to_map({user_type, Attrs, Name, Types}) -> %% any() - #{type => user_type, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - name => Name}, - content => to_map(Types)}; + #{ + type => type, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + name => Name + }, + content => to_map(Types) + }; +%% any() +to_map({user_type, Attrs, Name, Types}) -> + #{ + type => user_type, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + name => Name + }, + content => to_map(Types) + }; to_map({type, Attrs, map_field_assoc, Name, Type}) -> {Location, Text} = case Attrs of @@ -707,48 +905,71 @@ to_map({type, Attrs, map_field_assoc, Name, Type}) -> Attrs -> {get_location(Attrs), get_text(Attrs)} end, - #{type => type_map_field, - attrs => #{location => Location, text => Text}, - node_attrs => #{key => to_map(Name), type => to_map(Type)}}; + #{ + type => type_map_field, + attrs => #{location => Location, text => Text}, + node_attrs => #{key => to_map(Name), type => to_map(Type)} + }; to_map({remote_type, Attrs, [Module, Function, Args]}) -> - #{type => remote_type, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - node_attrs => - #{module => to_map(Module), - function => to_map(Function), - args => to_map(Args)}}; + #{ + type => remote_type, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => + #{ + module => to_map(Module), + function => to_map(Function), + args => to_map(Args) + } + }; to_map({ann_type, Attrs, [Var, Type]}) -> - #{type => record_field, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - node_attrs => #{var => to_map(Var), type => to_map(Type)}}; + #{ + type => record_field, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => #{var => to_map(Var), type => to_map(Type)} + }; to_map({paren_type, Attrs, [Type]}) -> - #{type => record_field, - attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, - node_attrs => #{type => to_map(Type)}}; -to_map(any) -> %% any() + #{ + type => record_field, + attrs => #{location => get_location(Attrs), text => get_text(Attrs)}, + node_attrs => #{type => to_map(Type)} + }; +%% any() +to_map(any) -> #{type => any}; %% Other Attributes to_map({attribute, Attrs, type, {Name, Type, Args}}) -> - #{type => type_attr, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - name => Name}, - node_attrs => #{args => to_map(Args), type => to_map(Type)}}; + #{ + type => type_attr, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + name => Name + }, + node_attrs => #{args => to_map(Args), type => to_map(Type)} + }; to_map({attribute, Attrs, spec, {{Name, Arity}, Types}}) -> - #{type => spec, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - name => Name, - arity => Arity}, - node_attrs => #{types => to_map(Types)}}; + #{ + type => spec, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + name => Name, + arity => Arity + }, + node_attrs => #{types => to_map(Types)} + }; to_map({attribute, Attrs, Type, Value}) -> - #{type => Type, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs), - value => Value}}; + #{ + type => Type, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs), + value => Value + } + }; %% Comments to_map({comment, Attrs, _Text}) -> #{type => comment, attrs => #{location => get_location(Attrs), text => get_text(Attrs)}}; @@ -762,12 +983,38 @@ to_map({macro, Attrs, Name, Args}) -> Args end, NameStr = macro_name(Name), - #{type => macro, - attrs => - #{location => get_location(Attrs), - text => get_text(Attrs) ++ NameStr, - name => NameStr}, - content => to_map(Args1)}; + #{ + type => macro, + attrs => + #{ + location => get_location(Attrs), + text => get_text(Attrs) ++ NameStr, + name => NameStr + }, + content => to_map(Args1) + }; +%% Representation of Parse Errors and End-of-File +to_map({error, E}) -> + #{ + type => error, + attrs => #{ + value => E + } + }; +to_map({warning, W}) -> + #{ + type => warning, + attrs => #{ + value => W + } + }; +to_map({eof, Location}) -> + #{ + type => eof, + attrs => #{ + location => get_location(Location) + } + }; %% Unhandled forms to_map(Parsed) when is_tuple(Parsed) -> case erl_syntax:is_tree(Parsed) of diff --git a/src/ktn_dodger.erl b/src/ktn_dodger.erl index c0cdadb..b8016e8 100644 --- a/src/ktn_dodger.erl +++ b/src/ktn_dodger.erl @@ -1,7 +1,17 @@ +%% erlfmt:ignore-begin %% ===================================================================== -%% Licensed under the Apache License, Version 2.0 (the "License"); you may -%% not use this file except in compliance with the License. You may obtain -%% a copy of the License at +%% %CopyrightBegin% +%% +%% SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later +%% +%% Copyright 2001-2006 Richard Carlsson +%% Copyright Ericsson AB 2009-2025. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 %% %% Unless required by applicable law or agreed to in writing, software %% distributed under the License is distributed on an "AS IS" BASIS, @@ -19,22 +29,12 @@ %% above, a recipient may use your version of this file under the terms of %% either the Apache License or the LGPL. %% -%% @copyright 2001-2006 Richard Carlsson +%% %CopyrightEnd% +%% %% @author Richard Carlsson %% @end %% ===================================================================== -%% @doc `epp_dodger' - bypasses the Erlang preprocessor. -%% -%%

This module tokenises and parses most Erlang source code without -%% expanding preprocessor directives and macro applications, as long as -%% these are syntactically "well-behaved". Because the normal parse -%% trees of the `erl_parse' module cannot represent these things -%% (normally, they are expanded by the Erlang preprocessor {@link -%% //stdlib/epp} before the parser sees them), an extended syntax tree -%% is created, using the {@link erl_syntax} module.

- - %% NOTES: %% %% * It's OK if the result does not parse - then at least nothing @@ -61,7 +61,7 @@ %% * We do our best to make macros without arguments pass the parsing %% stage transparently. Atoms are accepted in most contexts, but %% variables are not, so we use only atoms to encode these macros. -%% Sadly, the parsing sometimes discards even the line number info from +%% Sadly, the parsing sometimes discards even the location info from %% atom tokens, so we can only use the actual characters for this. %% %% * We recognize `?m(...' at the start of a form and prevent this from @@ -69,14 +69,25 @@ %% function definition. Likewise with attributes `-?m(...'. -module(ktn_dodger). - --format ignore. +%-moduledoc """ +%Bypassing the Erlang preprocessor. +% +%This module tokenises and parses most Erlang source code without expanding +%preprocessor directives and macro applications, as long as these are +%syntactically "well-behaved". Because the normal parse trees of the `erl_parse` +%module cannot represent these things (normally, they are expanded by the Erlang +%preprocessor [`//stdlib/epp`](`m:epp`) before the parser sees them), an extended +%syntax tree is created, using the `m:erl_syntax` module. +%""". + +-compile(nowarn_deprecated_catch). %% We have snake_case macros here -elvis([{elvis_style, macro_names, disable}]). -elvis([{elvis_style, no_catch_expressions, disable}]). -elvis([{elvis_style, no_throw, disable}]). -elvis([{elvis_style, consistent_variable_casing, disable}]). +-elvis([{elvis_style, nesting_level, disable}]). -export([parse_file/1, quick_parse_file/1, parse_file/2, quick_parse_file/2, parse/1, quick_parse/1, parse/2, quick_parse/2, parse/3, quick_parse/3, parse_form/2, parse_form/3, @@ -90,10 +101,6 @@ -define(var_prefix, "?,"). -define(pp_form, '?preprocessor declaration?'). - -%% This is a so-called Erlang I/O ErrorInfo structure; see the {@link -%% //stdlib/io} module for details. - -type errorinfo() :: erl_scan:error_info(). -type option() :: atom() | {atom(), term()}. @@ -102,68 +109,63 @@ -hank([{unnecessary_function_arguments, [{no_fix, 1}, {quick_parser, 2}]}]). -%% ===================================================================== -%% @equiv parse_file(File, []) - +%-doc #{equiv => parse_file(File, [])}. -spec parse_file(file:filename()) -> {ok, erl_syntax:forms()} | {error, errorinfo()}. parse_file(File) -> parse_file(File, []). -%% @doc Reads and parses a file. If successful, `{ok, Forms}' -%% is returned, where `Forms' is a list of abstract syntax -%% trees representing the "program forms" of the file (cf. -%% `erl_syntax:is_form/1'). Otherwise, `{error, errorinfo()}' is -%% returned, typically if the file could not be opened. Note that -%% parse errors show up as error markers in the returned list of -%% forms; they do not cause this function to fail or return -%% `{error, errorinfo()}'. -%% -%% Options: -%%
-%%
{@type {no_fail, boolean()@}}
-%%
If `true', this makes `epp_dodger' replace any program forms -%% that could not be parsed with nodes of type `text' (see {@link -%% erl_syntax:text/1}), representing the raw token sequence of the -%% form, instead of reporting a parse error. The default value is -%% `false'.
-%%
{@type {clever, boolean()@}}
-%%
If set to `true', this makes `epp_dodger' try to repair the -%% source code as it seems fit, in certain cases where parsing would -%% otherwise fail. Currently, it inserts `++'-operators between string -%% literals and macros where it looks like concatenation was intended. -%% The default value is `false'.
-%%
-%% -%% @see parse/2 -%% @see quick_parse_file/1 -%% @see erl_syntax:is_form/1 - +%-doc """ +%Reads and parses a file. +% +%If successful, `{ok, Forms}` is returned, where `Forms` is a list of +%abstract syntax trees representing the "program forms" of the file +%(see `erl_syntax:is_form/1`). Otherwise, `{error, errorinfo()}` is +%returned, typically if the file could not be opened. Note that parse +%errors show up as error markers in the returned list of forms; they do +%not cause this function to fail or return `{error, errorinfo()}`. +% +%Options: +% +%- **`{no_fail, boolean()}`** - If `true`, this makes `epp_dodger` replace any +% program forms that could not be parsed with nodes of type `text` (see +% `erl_syntax:text/1`), representing the raw token sequence of the form, instead +% of reporting a parse error. The default value is `false`. +% +%- **`{clever, boolean()}`** - If set to `true`, this makes `epp_dodger` try to +% repair the source code as it seems fit, in certain cases where parsing would +% otherwise fail. Currently, it inserts `++` operators between string literals +% and macros where it looks like concatenation was intended. The default value +% is `false`. +% +%_See also: _`parse/2`, `quick_parse_file/1`, `erl_syntax:is_form/1`. +%""". -spec parse_file(file:filename(), [option()]) -> {ok, erl_syntax:forms()} | {error, errorinfo()}. parse_file(File, Options) -> parse_file(File, fun parse/3, Options). -%% @equiv quick_parse_file(File, []) - +%-doc #{equiv => quick_parse_file(File, [])}. -spec quick_parse_file(file:filename()) -> {ok, erl_syntax:forms()} | {error, errorinfo()}. quick_parse_file(File) -> quick_parse_file(File, []). -%% @doc Similar to `parse_file/2', but does a more quick-and-dirty -%% processing of the code. Macro definitions and other preprocessor -%% directives are discarded, and all macro calls are replaced with -%% atoms. This is useful when only the main structure of the code is of -%% interest, and not the details. Furthermore, the quick-parse method -%% can usually handle more strange cases than the normal, more exact -%% parsing. -%% -%% Options: see {@link parse_file/2}. Note however that for -%% `quick_parse_file/2', the option `no_fail' is `true' by default. -%% -%% @see quick_parse/2 -%% @see parse_file/2 - +%-doc """ +%Similar to `parse_file/2`, but does a more quick-and-dirty processing of the +%code. +% +%Macro definitions and other preprocessor directives are discarded, and all +%macro calls are replaced with atoms. This is useful when only the main structure +%of the code is of interest, and not the details. Furthermore, the quick-parse +%method can usually handle more strange cases than the normal, more exact +%parsing. +% +%Options: see `parse_file/2`. However, note that for +%[`quick_parse_file/2`](`quick_parse_file/2`), the option `no_fail` is `true` by +%default. +% +%_See also: _`parse_file/2`, `quick_parse/2`. +%""". -spec quick_parse_file(file:filename(), [option()]) -> {ok, erl_syntax:forms()} | {error, errorinfo()}. quick_parse_file(File, Options) -> @@ -211,55 +213,49 @@ find_invalid_unicode([]) -> none. %% ===================================================================== -%% @equiv parse(IODevice, 1) +%-doc #{equiv => parse(IODevice, 1)}. -spec parse(file:io_device()) -> {ok, erl_syntax:forms()}. parse(Dev) -> parse(Dev, 1). -%% @equiv parse(IODevice, StartLocation, []) -%% @see parse/1 - +%-doc #{equiv => parse(IODevice, StartLocation, [])}. -spec parse(file:io_device(), erl_anno:location()) -> {ok, erl_syntax:forms()}. parse(Dev, L) -> parse(Dev, L, []). -%% @doc Reads and parses program text from an I/O stream. Characters are -%% read from `IODevice' until end-of-file; apart from this, the -%% behaviour is the same as for {@link parse_file/2}. `StartLocation' is the -%% initial location. -%% -%% @see parse/2 -%% @see parse_file/2 -%% @see parse_form/2 -%% @see quick_parse/3 - +%-doc """ +%Reads and parses program text from an I/O stream. +% +%Characters are read from `IODevice` until end-of-file; apart from +%this, the behavior is the same as for `parse_file/2`. `StartLocation` +%is the initial location. +% +%_See also: _`parse/2`, `parse_file/2`, `parse_form/2`, `quick_parse/3`. +%""". -spec parse(file:io_device(), erl_anno:location(), [option()]) -> {ok, erl_syntax:forms()}. parse(Dev, L0, Options) -> parse(Dev, L0, fun parse_form/3, Options). -%% @equiv quick_parse(IODevice, 1) - +%-doc #{equiv => quick_parse(IODevice, 1)}. -spec quick_parse(file:io_device()) -> {ok, erl_syntax:forms()}. quick_parse(Dev) -> quick_parse(Dev, 1). -%% @equiv quick_parse(IODevice, StartLocation, []) -%% @see quick_parse/1 - +%-doc #{equiv => quick_parse(IODevice, StartLocation, [])}. -spec quick_parse(file:io_device(), erl_anno:location()) -> {ok, erl_syntax:forms()}. quick_parse(Dev, L) -> quick_parse(Dev, L, []). -%% @doc Similar to `parse/3', but does a more quick-and-dirty -%% processing of the code. See `quick_parse_file/2' for details. -%% -%% @see quick_parse/2 -%% @see quick_parse_file/2 -%% @see quick_parse_form/2 -%% @see parse/3 - +%-doc """ +%Similar to `parse/3`, but does a more quick-and-dirty processing of the code. +% +%See `quick_parse_file/2` for details. +% +%_See also: _`parse/3`, `quick_parse/2`, `quick_parse_file/2`, +%`quick_parse_form/2`. +%""". -spec quick_parse(file:io_device(), erl_anno:location(), [option()]) -> {ok, erl_syntax:forms()}. quick_parse(Dev, L0, Options) -> parse(Dev, L0, fun quick_parse_form/3, Options). @@ -279,12 +275,7 @@ parse(Dev, L0, Fs, Parser, Options) -> {ok, lists:reverse(Fs)} end. - -%% ===================================================================== -%% @equiv parse_form(IODevice, StartLocation, []) -%% -%% @see quick_parse_form/2 - +%-doc #{equiv => parse_form(IODevice, StartLocation, [])}. -spec parse_form(file:io_device(), erl_anno:location()) -> {ok, erl_syntax:forms(), erl_anno:location()} | {eof, erl_anno:location()} | @@ -292,18 +283,18 @@ parse(Dev, L0, Fs, Parser, Options) -> parse_form(Dev, L0) -> parse_form(Dev, L0, []). -%% @doc Reads and parses a single program form from an I/O stream. -%% Characters are read from `IODevice' until an end-of-form -%% marker is found (a period character followed by whitespace), or until -%% end-of-file; apart from this, the behaviour is similar to that of -%% `parse/3', except that the return values also contain the -%% final location given that `StartLocation' is the initial -%% location, and that `{eof, Location}' may be returned. -%% -%% @see parse/3 -%% @see parse_form/2 -%% @see quick_parse_form/3 - +%-doc """ +%Reads and parses a single program form from an I/O stream. +% +%Characters are read from `IODevice` until an end-of-form marker is +%found (a period character followed by whitespace), or until +%end-of-file; apart from this, the behavior is similar to that of +%[`parse/3`](`parse/3`), except that the return values also contain the +%final location given that `StartLocation` is the initial location, and +%that `{eof, Location}` may be returned. +% +%_See also: _`parse/3`, `parse_form/2`, `quick_parse_form/3`. +%""". -spec parse_form(file:io_device(), erl_anno:location(), [option()]) -> {ok, erl_syntax:forms(), erl_anno:location()} | {eof, erl_anno:location()} | @@ -311,10 +302,7 @@ parse_form(Dev, L0) -> parse_form(Dev, L0, Options) -> parse_form(Dev, L0, fun normal_parser/2, Options). -%% @equiv quick_parse_form(IODevice, StartLocation, []) -%% -%% @see parse_form/2 - +%-doc #{equiv => quick_parse_form(IODevice, StartLocation, [])}. -spec quick_parse_form(file:io_device(), erl_anno:location()) -> {ok, erl_syntax:forms(), erl_anno:location()} | {eof, erl_anno:location()} | @@ -322,13 +310,12 @@ parse_form(Dev, L0, Options) -> quick_parse_form(Dev, L0) -> quick_parse_form(Dev, L0, []). -%% @doc Similar to `parse_form/3', but does a more quick-and-dirty -%% processing of the code. See `quick_parse_file/2' for details. -%% -%% @see parse/3 -%% @see quick_parse_form/2 -%% @see parse_form/3 - +%-doc """ +%Similar to `parse_form/3`, but does a more quick-and-dirty processing of the +%code. See `quick_parse_file/2` for details. +% +%_See also: _`parse/3`, `parse_form/3`, `quick_parse_form/2`. +%""". -spec quick_parse_form(file:io_device(), erl_anno:location(), [option()]) -> {ok, erl_syntax:forms(), erl_anno:location()} | {eof, erl_anno:location()} | @@ -424,10 +411,11 @@ start_pos([], L) -> parse_tokens(Ts) -> parse_tokens(Ts, fun no_fix/1, fun fix_form/1, fun no_fix/1). - -%% @doc PreFix adjusts the tokens before parsing them. -%% FormFix adjusts the tokens after parsing them, if erl_parse failed. -%% PostFix adjusts the forms after parsing them, if erl_parse worked. +%-doc """ +%PreFix adjusts the tokens before parsing them. +%FormFix adjusts the tokens after parsing them, if erl_parse failed. +%PostFix adjusts the forms after parsing them, if erl_parse worked. +%""". parse_tokens(Ts, PreFix, FormFix, PostFix) -> case PreFix(Ts) of {form, Form} -> @@ -455,9 +443,11 @@ parse_tokens(Ts, PreFix, FormFix, PostFix) -> end end. -%% @doc This handles config files, app.src, etc. -%% PreFix adjusts the tokens before parsing them. -%% FormFix adjusts the tokens after parsing them, only if erl_parse failed. +%-doc """ +%This handles config files, app.src, etc. +%PreFix adjusts the tokens before parsing them. +%FormFix adjusts the tokens after parsing them, only if erl_parse failed. +%""". parse_tokens_as_terms(Ts, PreFix, FormFix) -> case PreFix(Ts) of {form, Form} -> @@ -580,6 +570,8 @@ quick_macro_string(A) -> %% Skipping to the end of a macro call, tracking open/close constructs. +-spec skip_macro_args(Tokens :: term()) -> {Skipped :: list(), Rest :: term()}. + skip_macro_args([{'(', _} = T | Ts]) -> skip_macro_args(Ts, [')'], [T]); skip_macro_args(Ts) -> @@ -617,7 +609,6 @@ filter_form({function, _, ?pp_form, _, [{clause, _, [], [], [{atom, _, kill}]}]} filter_form(T) -> T. - %% --------------------------------------------------------------------- %% Normal parsing - try to preserve all information @@ -760,6 +751,8 @@ scan_macros([{'?', Anno}, {Type, _, _} = N | [{'(', _} | _] = Ts], [{':', _} | _ macro_call(Args, Anno, N, Rest, As, Opt); [{'when', _} | _] -> macro_call(Args, Anno, N, Rest, As, Opt); + [{':', _} | _] -> + macro_call(Args, Anno, N, Rest, As, Opt); _ -> macro(Anno, N, Ts, As, Opt) end; @@ -777,7 +770,7 @@ scan_macros([T | Ts], As, Opt) -> scan_macros([], As, _Opt) -> lists:reverse(As). -%% Rewriting to a call which will be recognized by the post-parse pass +%% Rewriting to a tuple which will be recognized by the post-parse pass %% (we insert parentheses to preserve the precedences when parsing). macro(Anno, {Type, _, A}, Rest, As, Opt) -> @@ -795,8 +788,13 @@ macro_call([{'(', _}, {')', _}], Anno, {_, AnnoN, _} = N, Rest, As, Opt) -> Opt); macro_call([{'(', _} | Args], L, {_, Ln, _} = N, Rest, As, Opt) -> {Open, Close} = parentheses(As), + %% drop closing parenthesis + + %% assert + {')', _} = lists:last(Args), + Args1 = lists:droplast(Args), %% note that we must scan the argument list; it may not be skipped - do_scan_macros(Args ++ Close, + do_scan_macros(Args1 ++ [{'}', Ln} | Close], Rest, lists:reverse(Open ++ [{atom, L, ?macro_call}, {'(', L}, N, {',', Ln}], As), Opt). @@ -871,6 +869,24 @@ rewrite(Node) -> _ -> do_rewrite(Node) end; + tuple -> + case erl_syntax:tuple_elements(Node) of + [MagicWord, A | As] -> + case erl_syntax:type(MagicWord) of + atom -> + case erl_syntax:atom_value(MagicWord) of + ?macro_call -> + M = erl_syntax:macro(A, rewrite_list(As)), + erl_syntax:copy_pos(Node, M); + _ -> + do_rewrite(Node) + end; + _ -> + do_rewrite(Node) + end; + _ -> + do_rewrite(Node) + end; _ -> do_rewrite(Node) end. @@ -974,9 +990,11 @@ fix_contiguous_strings([Other | Rest], Ts) -> no_fix(_) -> no_fix. -%% -%% @doc Generates a string corresponding to the given token sequence. -%% The string can be re-tokenized to yield the same token list again. +%-doc """ +%Generates a string corresponding to the given token sequence. +% +%The string can be re-tokenized to yield the same token list again. +%""". token_to_string(T) -> case erl_scan:text(T) of undefined -> @@ -1050,6 +1068,7 @@ maybe_space_between(_, _) -> %% @doc Callback function for formatting error descriptors. Not for %% normal use. +%-doc false. -spec format_error(term()) -> string(). format_error(macro_args) -> @@ -1064,12 +1083,13 @@ format_error({unknown, Reason}) -> errormsg(String) -> io_lib:format("~s: ~ts", [?MODULE, String]). - %% ===================================================================== -%% @doc The dodger currently does not process feature attributes -%% correctly, so temporarily consider the `else' and `maybe' atoms + +%% See #7266: The dodger currently does not process feature attributes +%% correctly, so temporarily consider the `else` and `maybe` atoms %% always as keywords -spec reserved_word(Atom :: atom()) -> boolean(). reserved_word('else') -> true; reserved_word('maybe') -> true; reserved_word(Atom) -> erl_scan:f_reserved_word(Atom). +%% erlfmt:ignore-end diff --git a/src/ktn_io_string.erl b/src/ktn_io_string.erl index 943c68a..93fc0fe 100644 --- a/src/ktn_io_string.erl +++ b/src/ktn_io_string.erl @@ -114,7 +114,7 @@ get_until(Module, Function, XArgs, Str) -> apply_get_until(Module, Function, [], Str, XArgs). -spec apply_get_until(module(), atom(), any(), string() | eof, list()) -> - {term(), string()}. + {term(), string()}. apply_get_until(Module, Function, State, String, XArgs) -> case apply(Module, Function, [State, String | XArgs]) of {done, Result, NewStr} -> @@ -124,12 +124,12 @@ apply_get_until(Module, Function, State, String, XArgs) -> end. -spec skip(string() | {cont, integer(), string()}, term(), integer()) -> - {more, {cont, integer(), string()}} | {done, integer(), string()}. + {more, {cont, integer(), string()}} | {done, integer(), string()}. skip(Str, _Data, Length) -> skip(Str, Length). -spec skip(string() | {cont, integer(), string()}, integer()) -> - {more, {cont, integer(), string()}} | {done, integer(), string()}. + {more, {cont, integer(), string()}} | {done, integer(), string()}. skip(Str, Length) when is_list(Str) -> {more, {cont, Length, Str}}; skip({cont, 0, Str}, Length) -> diff --git a/test/files/otp27.erl b/test/files/otp27.erl index 2e09d09..45d6c90 100644 --- a/test/files/otp27.erl +++ b/test/files/otp27.erl @@ -11,8 +11,11 @@ break() -> This is valid code. """), - Fun = fun () -> ok end, - ?assertMatch({ok, _} - when is_function(Fun, 0), {ok, 'no'}). + Fun = fun() -> ok end, + ?assertMatch( + {ok, _} when + is_function(Fun, 0), + {ok, 'no'} + ). -endif. diff --git a/test/files/otp28.erl b/test/files/otp28.erl new file mode 100644 index 0000000..785442b --- /dev/null +++ b/test/files/otp28.erl @@ -0,0 +1,37 @@ +-module(otp28). + +-if(?OTP_RELEASE >= 28). + +-moduledoc """ +Here we go +""". + +-export([valid/0]). + +%% erlfmt:ignore-begin + +-doc """ +There and back again. +""". +valid() -> + % list strict generator + [Integer || {Integer, _} <:- [{1, 2}, {3, 4}]], + + % binary strict generator + [Word || <> <:= <<16#1234:16, 16#ABCD:16>>], + + % map strict generator + #{K => V + 1 || K := V <:- #{a => 1, b => 2}}, + + % zip generators + [A + B || A <- [1, 2, 3] && B <- [4, 5, 6]], + + % binaries + %~"This is a UTF-8 binary", + + % map comprehensions + #{ K => V || K := V <- #{john => wick}}. + +%% erlfmt:ignore-end + +-endif. diff --git a/test/ktn_code_SUITE.erl b/test/ktn_code_SUITE.erl index d52644b..6bd2f54 100644 --- a/test/ktn_code_SUITE.erl +++ b/test/ktn_code_SUITE.erl @@ -1,10 +1,14 @@ -module(ktn_code_SUITE). -export([all/0, init_per_suite/1, end_per_suite/1]). --export([consult/1, beam_to_string/1, parse_tree/1, parse_tree_otp/1, latin1_parse_tree/1, - to_string/1]). - --if(?OTP_RELEASE >= 25). +-export([ + consult/1, + beam_to_string/1, + parse_tree/1, + parse_tree_otp/1, + latin1_parse_tree/1, + to_string/1 +]). -export([parse_maybe/1, parse_maybe_else/1]). @@ -12,6 +16,10 @@ -export([parse_sigils/1]). +-if(?OTP_RELEASE >= 28). + +-export([parse_generators/1]). + -endif. -endif. @@ -85,11 +93,15 @@ beam_to_string(_Config) -> -spec parse_tree(config()) -> ok. parse_tree(_Config) -> ModuleNode = - #{type => module, - attrs => - #{location => {1, 2}, - text => "module", - value => x}}, + #{ + type => module, + attrs => + #{ + location => {1, 2}, + text => "module", + value => x + } + }, #{type := root, content := _} = ktn_code:parse_tree("-module(x)."), @@ -122,9 +134,11 @@ latin1_parse_tree(_Config) -> error end, #{type := root, content := _} = - ktn_code:parse_tree(<<"%% -*- coding: latin-1 -*-\n" - "%% �" - "-module(x).">>), + ktn_code:parse_tree(<< + "%% -*- coding: latin-1 -*-\n" + "%% �" + "-module(x)." + >>), ok. @@ -136,14 +150,14 @@ to_string(_Config) -> ok. --if(?OTP_RELEASE >= 25). - -spec parse_maybe(config()) -> ok. parse_maybe(_Config) -> %% Note that to pass this test case, the 'maybe_expr' feature must be enabled. - #{type := root, - content := - [#{type := function, content := [#{type := clause, content := [#{type := 'maybe'}]}]}]} = + #{ + type := root, + content := + [#{type := function, content := [#{type := clause, content := [#{type := 'maybe'}]}]}] + } = ktn_code:parse_tree(<<"foo() -> maybe ok ?= ok end.">>), ok. @@ -151,9 +165,11 @@ parse_maybe(_Config) -> -spec parse_maybe_else(config()) -> ok. parse_maybe_else(_Config) -> %% Note that to pass this test case, the 'maybe_expr' feature must be enabled. - #{type := root, - content := - [#{type := function, content := [#{type := clause, content := [#{type := 'maybe'}]}]}]} = + #{ + type := root, + content := + [#{type := function, content := [#{type := clause, content := [#{type := 'maybe'}]}]}] + } = ktn_code:parse_tree(<<"foo() -> maybe ok ?= ok else _ -> ng end.">>), ok. @@ -162,8 +178,19 @@ parse_maybe_else(_Config) -> parse_sigils(_Config) -> {ok, _} = - ktn_dodger:parse_file("../../lib/katana_code/test/files/otp27.erl", - [no_fail, parse_macro_definitions]). + ktn_dodger:parse_file( + "../../lib/katana_code/test/files/otp27.erl", + [no_fail, parse_macro_definitions] + ). + +-if(?OTP_RELEASE >= 28). + +parse_generators(_Config) -> + {ok, _} = + ktn_dodger:parse_file( + "../../lib/katana_code/test/files/otp28.erl", + [no_fail] + ). -endif. -endif.