revng-revng/python/revng/internal/pipeline.yml

#
# This file is distributed under the MIT License. See LICENSE.md for details.
#

artifact-categories:
  # User-facing artifacts that the end-user is expected to see
  - name: user
    show-by-default: true
  # Artifacts that the user might request, but are machine-parsable and only
  # usable in conjunction with other software
  - name: automation
    show-by-default: false
  # Artifacts that the user is not expected to see but machine users
  # (e.g. scripting) are expected to use
  - name: metadata
    show-by-default: false
  # Debugging artifacts, these are for debugging purposes and/or testing
  - name: debug
    show-by-default: false

containers:
  - name: binaries-container
    type: BinariesContainer
  - name: llvm-root
    type: LLVMRootContainer
  - name: legacy-model-header
    type: PTMLCBytesContainer
  - name: legacy-model-types
    type: PTMLCTypeContainer
  - name: cfg-map
    type: CFGMap
  - name: llvm-functions
    type: LLVMFunctionContainer
  - name: hexdump
    type: HexDumpContainer
  - name: assembly-internal
    type: AssemblyInternalContainer
  - name: assembly
    type: AssemblyContainer
  - name: object-file
    type: ObjectFileContainer
  - name: translated
    type: TranslatedContainer
  - name: llvm-root-with-functions
    type: LLVMRootContainer
  - name: clift-functions
    type: CliftFunctionContainer
  - name: decompile-c
    type: PTMLCFunctionBytesContainer
  - name: decompiled-single-file
    type: PTMLCBytesContainer
  - name: tagless-decompile-c
    type: PTMLCFunctionBytesContainer
  - name: tagless-decompiled-single-file
    type: PTMLCBytesContainer
  - name: recompilable-archive
    type: RecompilableArchiveContainer
  - name: llvm-root-merged
    type: LLVMRootContainer
  - name: cross-relations
    type: CrossRelationsContainer
  - name: call-graph
    type: CallGraphContainer
  - name: call-graph-slice
    type: CallGraphSliceContainer
  - name: function-cfg
    type: FunctionControlFlowContainer
  - name: clift-types-and-globals
    type: CliftModuleContainer
  - name: type-and-global-header
    type: PTMLCBytesContainer
  - name: helper-header
    type: PTMLCBytesContainer
  - name: single-type
    type: PTMLCTypeBytesContainer
  - name: tagless-type-and-global-header
    type: PTMLCBytesContainer
  - name: tagless-helper-header
    type: PTMLCBytesContainer

analyses:
  - apply-diff
  - verify-diff
  - set-model
  - verify-model
  - import-prototypes-from-db
  - convert-functions-to-cabi
  - import-from-c

analysis-lists:
  - name: initial-auto-analysis
    analyses:
      - parse-binary
      - import-prototypes-from-db
      - detect-abi
      - detect-stack-size
      - analyze-data-layout
      - convert-functions-to-cabi
    description: |
      The initial auto analysis that should be run to automatically detect functions and types

branches:
  files-imported:
    tasks:
      - pipe: import-files
        arguments: [binaries-container]
        analyses:
          - analysis: parse-binary
            containers: [binaries-container]

  lift:
    from: files-imported
    tasks:
      - pipe: lift
        arguments: [binaries-container, llvm-root]
      - pipe: pure-llvm-passes-root-pipe
        arguments: [llvm-root]
        configuration:
          passes:
            - globaldce
      - savepoint: lifted
        containers: [llvm-root]
        artifacts:
          - name: lift
            container: llvm-root
            category: debug
            filename: lifted.bc
        analyses:
          - analysis: detect-abi
            containers: [llvm-root]

  collect-cfg:
    from: lift
    tasks:
      - pipe: collect-cfg
        arguments: [llvm-root, cfg-map]
      - savepoint: cfg-computed
        containers: [cfg-map]
        artifacts:
          - name: emit-cfg
            container: cfg-map
            category: debug

  isolate:
    from: collect-cfg
    tasks:
      - pipe: isolate
        arguments: [cfg-map, llvm-root, llvm-functions]
      - pipe: attach-debug-info
        arguments: [cfg-map, llvm-functions]
      - savepoint: isolate
        containers: [llvm-functions]
        artifacts:
          - name: isolate
            container: llvm-functions
            category: debug
            filename: isolated.bc

  enforce-abi:
    from: isolate
    tasks:
      - pipe: enforce-abi
        arguments: [cfg-map, llvm-functions]
      - pipe: pure-llvm-passes-pipe
        arguments: [llvm-functions]
        configuration:
          passes:
            - strip-debug-info-from-helpers
      # Note: we're running promote-csvs twice: it is important to run
      #       it before inline-helpers so that mem2reg can constant
      #       propagate helper arguments that are constant. This enables
      #       us to inline less code, in particular for helpers such as
      #       `cc_compute_c` and `cc_compute_all`.
      - pipe: promote-csvs
        arguments: [llvm-functions]
      - pipe: pure-llvm-passes-pipe
        arguments: [llvm-functions]
        configuration:
          passes:
            - mem2reg
            - link-helpers-to-inline
            - inline-helpers
            - delete-helper-bodies
      - pipe: attach-debug-info
        arguments: [cfg-map, llvm-functions]
      - pipe: promote-csvs
        arguments: [llvm-functions]
      - pipe: pure-llvm-passes-pipe
        arguments: [llvm-functions]
        configuration:
          passes:
            - remove-exceptional-functions
            - strip-dead-debug-info
      - savepoint: enforce-abi
        containers: [llvm-functions]
        artifacts:
          - name: enforce-abi
            container: llvm-functions
            category: debug
            filename: abi_enforced.bc

  detect-stack-size:
    from: enforce-abi
    tasks:
      - pipe: pure-llvm-passes-pipe
        arguments: [llvm-functions]
        configuration:
          passes: [dce]
      - pipe: remove-lifting-artifacts
        arguments: [llvm-functions]
      - pipe: promote-init-csv-to-undef
        arguments: [llvm-functions]
      - pipe: inject-stack-size-probes-at-callsites
        arguments: [llvm-functions]
      - pipe: promote-stack-pointer
        arguments: [llvm-functions]
      - pipe: pure-llvm-passes-pipe
        arguments: [llvm-functions]
        configuration:
          passes:
            - dce
            - remove-extractvalues
            - simplify-cfg-with-hoist-and-sink
            - dse
            - instcombine
            - remove-extractvalues
            - sroa
            - instsimplify
            - jump-threading
            - licm
            - unreachableblockelim
            - instcombine
            - remove-extractvalues
            - early-cse
            - simplify-cfg-with-hoist-and-sink
            - early-type-shrinking
            - type-shrinking
            - early-cse
            - instsimplify
            - gvn
            - instsimplify
            - dse
            - dce
            # In Outliner we inject some store of opaque values before calls
            # targeting the link register/top of the stack: the goal is make
            # sure the stores writing the return address are dead.
            # After the previous optimization pipeline, dead store elimination
            # should have removed such stores, therefore, we can now drop
            # those injected by us too.
            - drop-opaque-return-address
            - globaldce
      - pipe: simplify-switch
        arguments: [binaries-container, llvm-functions]
      - savepoint: simplify-switch
        containers: [llvm-functions]
        artifacts:
          - name: simplify-switch
            container: llvm-functions
            category: debug
            filename: simplify-switch.bc
      - pipe: pure-llvm-passes-pipe
        arguments: [llvm-functions]
        configuration:
          passes:
            - remove-stack-alignment
            - instrument-stack-accesses
            - instcombine
            - remove-extractvalues
            - loop-rotate
            - loop-simplify
            - compute-stack-accesses-bounds
      - savepoint: detect-stack-size
        containers: [llvm-functions]
        analyses:
          - analysis: detect-stack-size
            containers: [llvm-functions]

  make-segment-ref:
    from: detect-stack-size
    tasks:
      - pipe: pure-llvm-passes-pipe
        arguments: [llvm-functions]
        configuration:
          passes: [split-struct-phis]
      - pipe: legacy-segregate-stack-accesses
        arguments: [llvm-functions]
      - pipe: pure-llvm-passes-pipe
        arguments: [llvm-functions]
        configuration:
          passes:
            - cleanup-stack-size-markers
            - dce
            - sroa
            - instcombine
            - remove-extractvalues
            - sroa
            - simplify-cfg-with-hoist-and-sink
            # loop-rotate rearranges loops in a way that should make it
            # easier to match do-while loops
            - loop-rotate
            - loop-rewrite-with-canonical-induction-variable
            - simplify-cfg-with-hoist-and-sink
            # don't run simplify-cfg{,-with-hoist-and-sink} after
            # loop-simplify because it kills the loop-simplify form causing
            # DLA not to identify arrays properly
            - loop-simplify
            - instcombine
            - remove-extractvalues
            - early-cse
            - dce
            - strip-dead-prototypes
            - split-overflow-intrinsics
            - dce
            - remove-llvmassume-calls
      - pipe: make-segment-ref
        arguments: [binaries-container, llvm-functions]
      - savepoint: make-segment-ref
        containers: [llvm-functions]
        artifacts:
          - name: make-segment-ref
            container: llvm-functions
            category: debug
            filename: legacy_stack_accesses_segregated.bc
        analyses:
          - analysis: analyze-data-layout
            containers: [llvm-functions]

  clift-functions:
    from: detect-stack-size
    tasks:
      - pipe: pure-llvm-passes-pipe
        arguments: [llvm-functions]
        configuration:
          passes:
            - split-struct-phis
      - pipe: segregate-stack-accesses
        arguments: [llvm-functions]
      - pipe: pure-llvm-passes-pipe
        arguments: [llvm-functions]
        configuration:
          passes:
            - cleanup-stack-size-markers
            - dce
            - sroa-noarrays
            - instcombine-noarrays
            - remove-extractvalues
            - sroa-noarrays
            - simplify-cfg-with-hoist-and-sink
            # loop-rotate rearranges loops in a way that should make it
            # easier to match do-while loops
            - loop-rotate
            - loop-rewrite-with-canonical-induction-variable
            - simplify-cfg-with-hoist-and-sink
            # don't run simplify-cfg{,-with-hoist-and-sink} after
            # loop-simplify because it kills the loop-simplify form causing
            # DLA not to identify arrays properly
            - loop-simplify
            - instcombine-noarrays
            - remove-extractvalues
            - early-cse
            - dce
            - strip-dead-prototypes
            - split-overflow-intrinsics
            - dce
      - savepoint: segregate-stack-accesses
        containers: [llvm-functions]
        artifacts:
          - name: segregate-stack-accesses
            container: llvm-functions
            category: debug
            filename: stack_accesses_segregated.bc
      - pipe: pure-llvm-passes-pipe
        arguments: [llvm-functions]
        configuration:
          passes:
            - remove-llvmassume-calls
            - dce
            - peephole-opt-for-decompilation
            - ternary-reduction
            - exit-ssa
            - arithmetic-to-gep
            - extractvalue-to-gep
            - split-exponential-dataflow
      - pipe: switch-to-statements
        arguments: [llvm-functions]
      - pipe: pure-llvm-passes-pipe
        arguments: [llvm-functions]
        configuration:
          passes:
            - remove-constant-array-returns
            - dagify
            - inline-divergent-scopes
            - enforce-single-exit
            - materialize-trivial-goto
            - select-scope
            - inline-divergent-scopes
            - enforce-single-exit
            - materialize-trivial-goto
            - strip-dead-prototypes
            - defer-allocas
        artifacts:
          - name: clifter-input
            container: llvm-functions
            category: debug
            filename: clifter_input.bc
      - pipe: fix-pointer-size
        arguments: [llvm-functions]
      - pipe: clifter
        arguments: [llvm-functions, clift-functions]
        artifacts:
          - name: clift-unoptimized
            container: clift-functions
            category: debug
            filename: clifter_unoptimized.mlir.bc
      - pipe: pure-mlir-passes-pipe
        arguments: [clift-functions]
        configuration:
          passes:
            - trivial-return-elimination
            - label-merging
            - loop-detection
            - optimize-statements
            - label-merging
            - optimize-expressions
            - emit-field-accesses
            - canonicalize
            - optimize-expressions
            - terminal-branch-complement-hoisting
            - tighten-variable-scopes
            - hoist-variable-initializers
        artifacts:
          - name: clift-optimized
            container: clift-functions
            category: debug
            filename: clift_optimized.mlir.bc
      - pipe: import-function-data-model
        arguments: [clift-functions]
      - pipe: pure-mlir-passes-pipe
        arguments: [clift-functions]
        configuration:
          passes:
            - c-legalization
            - elide-implicit-casts
            - deduce-immediate-radices
      - pipe: verify-function-against-model
        arguments: [clift-functions]
      - savepoint: decompile-clift
        containers: [clift-functions]
      - pipe: import-descriptive-function-info
        arguments: [clift-functions]
      - savepoint: pre-backend-clift
        containers: [clift-functions]
        artifacts:
          - name: pre-backend-clift
            container: clift-functions
            category: debug
            filename: pre_backend_clift.mlir.bc

  decompiled:
    from: clift-functions
    tasks:
      - pipe: emit-c
        arguments: [clift-functions, decompile-c]
      - savepoint: emit-c
        containers: [decompile-c]
        artifacts:
          - name: emit-c
            container: decompile-c
            category: user
            filename: decompiled.c
            defined_locations:
              - function
            preferred_artifacts:
              - emit-type-and-global-header
        analyses:
          - analysis: llm-rename
            containers: [decompile-c]

  helpers:
    from: clift-functions
    tasks:
      - pipe: emit-helper-header
        arguments: [clift-functions, helper-header]
      - savepoint: helper-header
        containers: [helper-header]
        artifacts:
          - name: emit-helper-header
            container: helper-header
            category: user
            filename: helpers.h
            defined_locations:
              - helper-function
              - helper-struct-type
              - helper-struct-field

  single-file:
    from: decompiled
    tasks:
      - pipe: emit-c-as-single-file
        arguments: [decompile-c, decompiled-single-file]
      - savepoint: emit-c-as-single-file
        containers: [decompiled-single-file]
        artifacts:
          - name: emit-c-as-single-file
            container: decompiled-single-file
            category: user
            filename: binary_decompiled.c
            defined_locations:
              - function
            preferred_artifacts:
              - emit-type-and-global-header

  tagless-decompiled:
    from: clift-functions
    tasks:
      - pipe: emit-c
        configuration:
          disable-markup: true
          emission-mode: recompilable
        arguments: [clift-functions, tagless-decompile-c]
      - savepoint: emit-tagless-c
        containers: [tagless-decompile-c]
        artifacts:
          - name: emit-tagless-c
            container: tagless-decompile-c
            category: metadata
            filename: decompiled.c
            defined_locations:
              - function
            preferred_artifacts:
              - emit-type-and-global-header

  tagless-single-file:
    from: tagless-decompiled
    tasks:
      - pipe: emit-c-as-single-file
        arguments: [tagless-decompile-c, tagless-decompiled-single-file]
        configuration:
          disable-markup: true
          emission-mode: recompilable
      - savepoint: emit-tagless-c-as-single-file
        containers: [tagless-decompiled-single-file]
        artifacts:
          - name: emit-tagless-c-as-single-file
            container: tagless-decompiled-single-file
            category: metadata
            filename: binary_decompiled.c
            defined_locations:
              - function
            preferred_artifacts:
              - emit-type-and-global-header

      - pipe: emit-helper-header
        arguments: [clift-functions, tagless-helper-header]
        configuration:
          disable-markup: true
          emission-mode: recompilable
        artifacts:
          - name: emit-tagless-helper-header
            container: tagless-helper-header
            category: metadata
            filename: helpers.h

      - pipe: import-types
        arguments: [clift-types-and-globals]
      - pipe: import-function-declarations
        arguments: [clift-types-and-globals]
      - pipe: import-segment-declarations
        arguments: [clift-types-and-globals]
      - pipe: import-descriptive-info
        arguments: [clift-types-and-globals]
      - pipe: emit-type-and-global-header
        arguments: [clift-types-and-globals, tagless-type-and-global-header]
        configuration:
          disable-markup: true
          emission-mode: recompilable
        artifacts:
          - name: emit-tagless-type-and-global-header
            container: tagless-type-and-global-header
            category: metadata
            filename: types-and-globals.h

      - pipe: emit-c-as-directory
        arguments:
          - tagless-decompiled-single-file
          - tagless-type-and-global-header
          - tagless-helper-header
          - recompilable-archive
      - savepoint: recompilable-archive
        containers: [recompilable-archive]
        artifacts:
          - name: emit-recompilable-archive
            container: recompilable-archive
            category: metadata
            filename: recompilable_archive.tar

  legacy-emit-model-header:
    tasks:
      - pipe: legacy-model-to-header
        arguments: [legacy-model-header]
      - savepoint: legacy-model-header
        containers: [legacy-model-header]
        artifacts:
          - name: legacy-emit-model-header
            container: legacy-model-header
            category: user
            filename: types-and-globals.h
            defined_locations:
              - type-definition
              - struct-field
              - union-field
              - enum-entry
              - dynamic-function
              - segment
              - artificial-struct
            preferred_artifacts:
              - emit-c

  legacy-emit-type-definition:
    tasks:
      - pipe: legacy-generate-model-type-definition
        arguments: [legacy-model-types]
      - savepoint: legacy-model-types
        containers: [legacy-model-types]
        artifacts:
          - name: legacy-emit-type-definitions
            container: legacy-model-types
            category: user
            filename: type.h

  hexdump:
    from: isolate
    tasks:
      - pipe: hex-dump
        arguments:
          - binaries-container
          - llvm-functions
          - cfg-map
          - hexdump
      - savepoint: hexdump
        containers: [hexdump]
        artifacts:
          - name: hexdump
            container: hexdump
            category: user
            filename: hex_dump.hex

  process-assembly:
    from: collect-cfg
    tasks:
      - pipe: process-assembly
        arguments: [binaries-container, cfg-map, assembly-internal]

  disassemble:
    from: process-assembly
    tasks:
      - pipe: yield-assembly
        arguments: [assembly-internal, assembly]
      - savepoint: disassemble
        containers: [assembly]
        artifacts:
          - name: disassemble
            container: assembly
            category: user
            filename: disassembly.S
            defined_locations:
              - function
              - basic-block
              - instruction

  recompile:
    from: lift
    tasks:
      - pipe: link-support
        arguments: [llvm-root]
      - pipe: pure-llvm-passes-root-pipe
        arguments: [llvm-root]
        configuration:
          passes: [drop-opaque-return-address]
      - pipe: compile-root-module
        arguments: [llvm-root, object-file]
      - pipe: link-for-translation
        arguments: [binaries-container, object-file, translated]
      - savepoint: recompile
        containers: [translated]
        artifacts:
          - name: recompile
            container: translated
            category: automation
            filename: translated-binary

  recompile-isolated:
    from: isolate
    tasks:
      - pipe: invoke-isolated-functions
        arguments: [llvm-root, llvm-functions, llvm-root-with-functions]
      - pipe: link-support
        arguments: [llvm-root-with-functions]
      - pipe: pure-llvm-passes-root-pipe
        arguments: [llvm-root-with-functions]
        configuration:
          passes: [drop-opaque-return-address]
      - pipe: compile-root-module
        arguments: [llvm-root-with-functions, object-file]
      - pipe: link-for-translation
        arguments: [binaries-container, object-file, translated]
      - savepoint: recompile-isolated
        containers: [translated]
        artifacts:
          - name: recompile-isolated
            container: translated
            category: automation
            filename: isolated-translated-binary

  cleanup-ir:
    from: make-segment-ref
    tasks:
      - pipe: merge-llvm-modules
        arguments: [llvm-functions, llvm-root-merged]
      - pipe: pure-llvm-passes-root-pipe
        arguments: [llvm-root-merged]
        configuration:
          passes: [instcombine]
      - pipe: cleanup-ir
        arguments: [llvm-root-merged]
      - pipe: pure-llvm-passes-root-pipe
        arguments: [llvm-root-merged]
        configuration:
          passes: [dce]
      - savepoint: cleanup-ir
        containers: [llvm-root-merged]
        artifacts:
          - name: cleanup-ir
            container: llvm-root-merged
            category: automation
            filename: clean-ir.bc

  process-call-graph:
    from: collect-cfg
    tasks:
      - pipe: process-call-graph
        arguments: [cfg-map, cross-relations]
        artifacts:
          - name: cross-relations
            container: cross-relations
            category: metadata
            filename: cross-relations.yml

  yield-call-graph:
    from: process-call-graph
    tasks:
      - pipe: yield-call-graph
        arguments: [cross-relations, call-graph]
      - savepoint: yield-call-graph
        containers: [call-graph]
        artifacts:
          - name: render-svg-call-graph
            container: call-graph
            category: user
            filename: call-graph.svg

  yield-call-graph-slice:
    from: process-call-graph
    tasks:
      - pipe: yield-call-graph-slice
        arguments: [cross-relations, call-graph-slice]
      - savepoint: yield-call-graph-slice
        containers: [call-graph-slice]
        artifacts:
          - name: render-svg-call-graph-slice
            container: call-graph-slice
            category: user
            filename: call-graph-slice.svg

  yield-cfg:
    from: process-assembly
    tasks:
      - pipe: yield-cfg
        arguments: [assembly-internal, function-cfg]
      - savepoint: yield-cfg
        containers: [function-cfg]
        artifacts:
          - name: render-svg-cfg
            container: function-cfg
            category: user
            filename: cfg.svg

  import-types:
    tasks:
      - pipe: import-types
        arguments: [clift-types-and-globals]
        artifacts:
          - name: import-types
            container: clift-types-and-globals
            category: debug
            filename: types.mlir
      - pipe: import-function-declarations
        arguments: [clift-types-and-globals]

  emit-type-and-global-header:
    from: import-types
    tasks:
      - pipe: import-segment-declarations
        arguments: [clift-types-and-globals]
      - pipe: import-descriptive-info
        arguments: [clift-types-and-globals]
        artifacts:
          - name: type-and-global-header-clift
            container: clift-types-and-globals
            category: debug
            filename: type-and-global-header.mlir.bc
      - pipe: emit-type-and-global-header
        arguments: [clift-types-and-globals, type-and-global-header]
        configuration:
          disable-markup: false
          emission-mode: recompilable
      - savepoint: type-and-global-header
        containers: [type-and-global-header]
        artifacts:
          - name: emit-type-and-global-header
            container: type-and-global-header
            category: user
            filename: types-and-globals.h
            defined_locations:
              - type-definition
              - struct-field
              - union-field
              - enum-entry
              - dynamic-function
              - segment
              - artificial-struct
            preferred_artifacts:
              - emit-c

  emit-single-type-definition:
    from: import-types
    tasks:
      - pipe: import-descriptive-info
        arguments: [clift-types-and-globals]
      - pipe: emit-single-type-definition
        arguments: [clift-types-and-globals, single-type]
        configuration:
          disable-markup: true
          emission-mode: editable
      - savepoint: single-type-definition
        containers: [single-type]
        artifacts:
          - name: emit-single-type-definition
            container: single-type
            category: user
            filename: type.h