preprocessor_build.hpp

Go to the documentation of this file.

 
#pragma once
 
#include <arkanjo/methods/diff/diff_method.hpp>
#include <arkanjo/methods/tfidf/tfidf_method.hpp>
#include <arkanjo/methods/ast/ast_method.hpp>
#include <arkanjo/methods/embedding/embedding_method.hpp>
 
#include "function_breaker.hpp"
#include <arkanjo/base/preprocess_state.hpp>
#include <arkanjo/commands/command_base.hpp>
#include <optional>
#include <string>
#include <tuple>
#include <filesystem>
 
namespace fs = std::filesystem;
 
using MethodFactory = std::function<std::unique_ptr<IMethod>(
  const std::string&, float, const std::vector<std::string>&
)>;
 
struct MethodInfo {
  size_t id;
  std::string_view name;
  MethodFactory create;
  std::string_view description;
};
 
class PreprocessorBuild : public Preprocess_State, public CommandBase<PreprocessorBuild> {
  private:
    static constexpr const char* TEMPLATE_COUNT =
      "The total number of functions that are "
      "similar to the found one is {count}. "
      "More info about them are listed below.";
    // User interaction messages
    static constexpr const char* PROJECT_PATH_MESSAGE = "Enter your project path:";                                  
    static constexpr const char* MINIMUM_SIMILARITY_MESSAGE = "Enter minimum similarity desired on using the tool:"; 
 
    // Processing stage messages
    static constexpr const char* INITIAL_MESSAGE = "Initiating Preprocessing";                                           
    static constexpr const char* BREAKER_MESSAGE = "Reading codebase... (this may take a while)";                        
    static constexpr const char* DUPLICATION_MESSAGE = "Finding duplication in the codebase... (this may take a while)"; 
    static constexpr const char* SAVING_MESSAGE = "Saving results...";                                                   
    static constexpr const char* END_MESSAGE = "Finished preprocessing";                                                 
    static constexpr const char* ERROR_PATH_MESSAGE = "Provided path does not exist.";                 
 
    // Duplication finder selection messages
    static constexpr const char* MESSAGE_DUPLICATION_FINDER_TYPE = "Enter the number of the duplication finder technique you want to use:";
    static constexpr const char* INVALID_CODE_DUPLICATION_FINDER = "Invalid option entered. Stopping Program...";
 
    bool mode_verbose = false; 
    int minimum_lines = 0;     
 
    const std::vector<MethodInfo> MethodsType = {
      {
        1,
        "tfidf",
        [](const std::string& base_path, float similarity,
           const std::vector<std::string>&) {
          return std::make_unique<TfidfMethod>(base_path, similarity);
        },
        "NLP text similarity using gensim"
      },
      {
        2,
        "diff",
        [](const std::string& base_path, float similarity,
           const std::vector<std::string>&) {
          return std::make_unique<DiffMethod>(base_path, similarity);
        },
        "Count proportion of equal lines using diff command"
      },
      {
        3,
        "ast",
        [](const std::string& base_path, float similarity,
           const std::vector<std::string>&) {
          return std::make_unique<ASTMethod>(base_path, similarity);
        },
        "Compare linearized structural sequences extracted from Tree-sitter ASTs"
      },
      {
        4,
        "embedding",
        [](const std::string& base_path, float similarity,
           const std::vector<std::string>& pass_through_args) {
          return std::make_unique<EmbeddingMethod>(base_path, similarity, pass_through_args);
        },
        "Embedding-based similarity using a code language model"
      }
    };
 
    std::tuple<std::string, double, size_t, std::vector<std::string>, Granularity>
    read_parameters(const ParsedOptions& options);
 
    void preprocess(const fs::path& path, double similarity, size_t use_duplication_finder_index,
                    const std::vector<std::string>& pass_through_args = {},
                    Granularity granularity = Granularity::Function);
 
  public:
    static constexpr CliOption options_[] = {
      {"verbose", 0, NoArgument, "Enable verbose output"},
      {"path", 0, RequiredArgument, "Project path to preprocess."},
      {"minimum-lines", 0, RequiredArgument, "Minimum clone size in original lines."},
      {"granularity", 0, RequiredArgument,
        "Comparison granularity: 'function' (default) compares individual "
        "functions; 'file' keeps each file whole and compares files against "
        "each other. Applies to every duplication finder method."},
      {"method", 0, RequiredArgument,
        "Duplication finder method: 1|tfidf, 2|diff, 3|ast, 4|embedding."},
      OPTION_END
    };
 
    PreprocessorBuild() = default;
 
    PreprocessorBuild(bool force_preprocess, const fs::path& path, double similarity);
 
    COMMAND_DESCRIPTION("Creates the foundation for fast query responses during the main operation phase.")
 
    bool validate(const ParsedOptions& options) override;
 
    bool run(const ParsedOptions& options) override;
};