[goalc] reduce compiler memory usage (#2247)

common/goal_constants.h

@@ -61,7 +61,7 @@ constexpr s32 max_symbols(GameVersion version) {
   }
 }
 
-enum class RegClass { GPR_64, FLOAT, INT_128, VECTOR_FLOAT, INVALID };
+enum class RegClass : u8 { GPR_64, FLOAT, INT_128, VECTOR_FLOAT, INVALID };
 
 constexpr u32 GOAL_NEW_METHOD = 0;       // method ID of GOAL new
 constexpr u32 GOAL_DEL_METHOD = 1;       // method ID of GOAL delete

common/goos/TextDB.cpp

@@ -175,39 +175,45 @@ std::optional<TextDb::ShortInfo> TextDb::get_short_info_for(const std::shared_pt
   return std::make_optional(info_result);
 }
 
+std::optional<TextDb::ShortInfo> TextDb::try_get_short_info(
+    const std::shared_ptr<goos::HeapObject>& heap_obj) const {
+  auto it = m_map.find(heap_obj);
+  if (it != m_map.end()) {
+    auto& frag = it->second.frag;
+    // shorten the string
+    std::string name = frag->get_description();
+    size_t start = 0;
+    for (size_t i = 0; i < name.size(); i++) {
+      if (name[i] == '/' || name[i] == '\\') {
+        start = i + 1;
+      }
+    }
+    if (start < name.size()) {
+      name = name.substr(start);
+    }
+
+    ShortInfo result;
+    result.filename = name;
+
+    int line_idx = frag->get_line_idx(it->second.offset);
+    result.line_idx_to_display = line_idx + 1;
+
+    int offset_of_line = frag->get_offset_of_line(line_idx);
+    int offset_of_next_line = frag->get_offset_of_line(line_idx + 1);
+
+    int line_length = offset_of_next_line - offset_of_line;
+
+    int start_offset_in_line = it->second.offset - offset_of_line - 1;
+    result.pos_in_line = std::max(start_offset_in_line, 0);
+    result.line_text = std::string(frag->get_text() + offset_of_line + 1, line_length - 1);
+    return result;
+  }
+  return {};
+}
+
 std::optional<TextDb::ShortInfo> TextDb::try_get_short_info(const Object& o) const {
   if (o.is_pair()) {
-    auto it = m_map.find(o.heap_obj);
-    if (it != m_map.end()) {
-      auto& frag = it->second.frag;
-      // shorten the string
-      std::string name = frag->get_description();
-      size_t start = 0;
-      for (size_t i = 0; i < name.size(); i++) {
-        if (name[i] == '/' || name[i] == '\\') {
-          start = i + 1;
-        }
-      }
-      if (start < name.size()) {
-        name = name.substr(start);
-      }
-
-      ShortInfo result;
-      result.filename = name;
-
-      int line_idx = frag->get_line_idx(it->second.offset);
-      result.line_idx_to_display = line_idx + 1;
-
-      int offset_of_line = frag->get_offset_of_line(line_idx);
-      int offset_of_next_line = frag->get_offset_of_line(line_idx + 1);
-
-      int line_length = offset_of_next_line - offset_of_line;
-
-      int start_offset_in_line = it->second.offset - offset_of_line - 1;
-      result.pos_in_line = std::max(start_offset_in_line, 0);
-      result.line_text = std::string(frag->get_text() + offset_of_line + 1, line_length - 1);
-      return result;
-    }
+    return try_get_short_info(o.heap_obj);
   }
 
   return {};
@@ -245,4 +251,9 @@ void TextDb::inherit_info(const Object& parent, const Object& child) {
     }
   }
 }
+
+void TextDb::clear_info() {
+  m_map.clear();
+  m_fragments.clear();
+}
 }  // namespace goos

common/goos/TextDB.h

@@ -112,8 +112,11 @@ class TextDb {
   std::optional<ShortInfo> get_short_info_for(const std::shared_ptr<SourceText>& frag,
                                               int offset) const;
   std::optional<ShortInfo> try_get_short_info(const Object& o) const;
+  std::optional<ShortInfo> try_get_short_info(const std::shared_ptr<goos::HeapObject>& o) const;
+
   bool has_info(const Object& o) const;
   void inherit_info(const Object& parent, const Object& child);
+  void clear_info();
 
  private:
   std::vector<std::shared_ptr<SourceText>> m_fragments;

common/type_system/TypeSpec.h

@@ -146,7 +146,7 @@ class TypeSpec {
     }
   }
 
-  const cu::SmallVector<TypeTag, 1>& tags() const { return m_tags; }
+  const std::vector<TypeTag>& tags() const { return m_tags; }
 
  private:
   friend class TypeSystem;
@@ -154,5 +154,5 @@ class TypeSpec {
   // hiding this behind a pointer makes things faster in the case where we have no
   // arguments (most of the time) and makes the type analysis pass in the decompiler 2x faster.
   std::vector<TypeSpec>* m_arguments = nullptr;
-  cu::SmallVector<TypeTag, 1> m_tags;
+  std::vector<TypeTag> m_tags;
 };

goalc/compiler/CodeGenerator.cpp

@@ -38,7 +38,12 @@ std::vector<u8> CodeGenerator::run(const TypeSystem* ts) {
     }
     auto rec =
         m_gen.add_function_to_seg(f->segment, &m_debug_info->add_function(f->name(), m_fe->name()));
-    rec.debug->function = f;
+    for (auto& x : f->code_source()) {
+      rec.debug->code_sources.push_back(x.heap_obj);
+    }
+    for (auto& x : f->code()) {
+      rec.debug->ir_strings.push_back(x->print());
+    }
   }
 
   // next, add all static objects.
@@ -48,14 +53,14 @@ std::vector<u8> CodeGenerator::run(const TypeSystem* ts) {
 
   // next, add instructions to functions
   for (size_t i = 0; i < m_fe->functions().size(); i++) {
-    do_function(m_fe->functions().at(i), i);
+    do_function(m_fe->functions().at(i).get(), i);
   }
 
   // generate a v3 object.
   return m_gen.generate_data_v3(ts).to_vector();
 }
 
-void CodeGenerator::do_function(const std::shared_ptr<FunctionEnv>& env, int f_idx) {
+void CodeGenerator::do_function(FunctionEnv* env, int f_idx) {
   if (env->is_asm_func) {
     do_asm_function(env, f_idx, env->asm_func_saved_regs);
   } else {
@@ -67,7 +72,7 @@ void CodeGenerator::do_function(const std::shared_ptr<FunctionEnv>& env, int f_i
  * Add instructions to the function, specified by index.
  * Generates prologues / epilogues.
  */
-void CodeGenerator::do_goal_function(const std::shared_ptr<FunctionEnv>& env, int f_idx) {
+void CodeGenerator::do_goal_function(FunctionEnv* env, int f_idx) {
   bool use_new_xmms = true;
   auto* debug = &m_debug_info->function_by_name(env->name());
 
@@ -275,9 +280,7 @@ void CodeGenerator::do_goal_function(const std::shared_ptr<FunctionEnv>& env, in
   m_gen.add_instr_no_ir(f_rec, IGen::ret(), InstructionInfo::Kind::EPILOGUE);
 }
 
-void CodeGenerator::do_asm_function(const std::shared_ptr<FunctionEnv>& env,
-                                    int f_idx,
-                                    bool allow_saved_regs) {
+void CodeGenerator::do_asm_function(FunctionEnv* env, int f_idx, bool allow_saved_regs) {
   auto f_rec = m_gen.get_existing_function_record(f_idx);
   const auto& allocs = env->alloc_result();
 

goalc/compiler/CodeGenerator.h

@@ -23,9 +23,9 @@ class CodeGenerator {
   emitter::ObjectGeneratorStats get_obj_stats() const { return m_gen.get_stats(); }
 
  private:
-  void do_function(const std::shared_ptr<FunctionEnv>& env, int f_idx);
-  void do_goal_function(const std::shared_ptr<FunctionEnv>& env, int f_idx);
-  void do_asm_function(const std::shared_ptr<FunctionEnv>& env, int f_idx, bool allow_saved_regs);
+  void do_function(FunctionEnv* env, int f_idx);
+  void do_goal_function(FunctionEnv* env, int f_idx);
+  void do_asm_function(FunctionEnv* env, int f_idx, bool allow_saved_regs);
   emitter::ObjectGenerator m_gen;
   FileEnv* m_fe = nullptr;
   DebugInfo* m_debug_info = nullptr;

goalc/compiler/Compiler.cpp

@@ -304,6 +304,7 @@ std::vector<u8> Compiler::codegen_object_file(FileEnv* env) {
     }
     auto stats = gen.get_obj_stats();
     m_debug_stats.num_moves_eliminated += stats.moves_eliminated;
+    env->cleanup_after_codegen();
     return result;
   } catch (std::exception& e) {
     throw_compiler_error_no_code("Error during codegen: {}", e.what());

goalc/compiler/Compiler.h

@@ -108,7 +108,7 @@ class Compiler {
   std::unordered_map<std::string, goos::ArgumentSpec> m_macro_specs;
   std::unordered_map<std::string, TypeSpec> m_symbol_types;
   std::unordered_map<goos::HeapObject*, goos::Object> m_global_constants;
-  std::unordered_map<goos::HeapObject*, LambdaVal*> m_inlineable_functions;
+  std::unordered_map<goos::HeapObject*, InlineableFunction> m_inlineable_functions;
   CompilerSettings m_settings;
   bool m_throw_on_define_extern_redefinition = false;
   std::unordered_set<std::string> m_allow_inconsistent_definition_symbols;
@@ -705,6 +705,7 @@ class Compiler {
                                          const goos::Object& rest,
                                          Env* env);
   Val* compile_go_hook(const goos::Object& form, const goos::Object& rest, Env* env);
+  Val* compile_gc_text(const goos::Object& form, const goos::Object& rest, Env* env);
 };
 
 extern const std::unordered_map<

goalc/compiler/Env.cpp

@@ -179,6 +179,14 @@ bool FileEnv::is_empty() {
   return m_functions.size() == 1 && m_functions.front().get() == m_top_level_func &&
          m_top_level_func->code().empty();
 }
+
+void FileEnv::cleanup_after_codegen() {
+  m_top_level_func = nullptr;
+  m_functions.clear();
+  m_statics.clear();
+  m_vals.clear();
+}
+
 ///////////////////
 // FunctionEnv
 ///////////////////

goalc/compiler/Env.h

@@ -102,7 +102,7 @@ class FileEnv : public Env {
   void add_top_level_function(std::unique_ptr<FunctionEnv> fe);
   void add_static(std::unique_ptr<StaticObject> s);
   void debug_print_tl();
-  const std::vector<std::shared_ptr<FunctionEnv>>& functions() { return m_functions; }
+  const std::vector<std::unique_ptr<FunctionEnv>>& functions() { return m_functions; }
   const std::vector<std::unique_ptr<StaticObject>>& statics() { return m_statics; }
   std::string get_anon_function_name() {
     return "anon-function-" + std::to_string(m_anon_func_counter++);
@@ -128,9 +128,11 @@ class FileEnv : public Env {
   void set_debug_file() { m_default_segment = DEBUG_SEGMENT; }
   bool is_debug_file() const { return default_segment() == DEBUG_SEGMENT; }
 
+  void cleanup_after_codegen();
+
  protected:
   std::string m_name;
-  std::vector<std::shared_ptr<FunctionEnv>> m_functions;
+  std::vector<std::unique_ptr<FunctionEnv>> m_functions;
   std::vector<std::unique_ptr<StaticObject>> m_statics;
   int m_anon_func_counter = 0;
   std::vector<std::unique_ptr<Val>> m_vals;

goalc/compiler/Lambda.h

@@ -1,8 +1,5 @@
 #pragma once
 
-#ifndef JAK_LAMBDA_H
-#define JAK_LAMBDA_H
-
 #include "common/goos/Object.h"
 #include "common/type_system/TypeSpec.h"
 
@@ -19,4 +16,8 @@ struct Lambda {
   goos::Object body;
 };
 
-#endif  // JAK_LAMBDA_H
+struct InlineableFunction {
+  Lambda lambda;
+  TypeSpec type;
+  bool inline_by_default = false;
+};

goalc/compiler/Val.cpp

@@ -158,7 +158,7 @@ RegVal* LambdaVal::to_reg(const goos::Object& form, Env* fe) {
 
 RegVal* InlinedLambdaVal::to_reg(const goos::Object& form, Env* fe) {
   throw std::runtime_error("Cannot put InlinedLambdaVal in a register.");
-  return lv->to_reg(form, fe);
+  return nullptr;
 }
 
 RegVal* FloatConstantVal::to_reg(const goos::Object& form, Env* fe) {

goalc/compiler/Val.h

@@ -140,9 +140,9 @@ class LambdaVal : public Val {
 
 class InlinedLambdaVal : public Val {
  public:
-  explicit InlinedLambdaVal(TypeSpec ts, LambdaVal* _lv) : Val(std::move(ts)), lv(_lv) {}
-  std::string print() const override { return "inline-lambda-" + lv->lambda.debug_name; }
-  LambdaVal* lv = nullptr;
+  explicit InlinedLambdaVal(TypeSpec ts, InlineableFunction _lv) : Val(std::move(ts)), lv(_lv) {}
+  std::string print() const override { return "inline-lambda-" + lv.lambda.debug_name; }
+  InlineableFunction lv;
   RegVal* to_reg(const goos::Object& form, Env* fe) override;
 };
 

goalc/compiler/compilation/Atoms.cpp

@@ -160,6 +160,7 @@ const std::unordered_map<
         {"make", {"", &Compiler::compile_make}},
         {"print-debug-compiler-stats", {"", &Compiler::compile_print_debug_compiler_stats}},
         {"gen-docs", {"", &Compiler::compile_gen_docs}},
+        {"gc-text", {"", &Compiler::compile_gc_text}},
 
         // CONDITIONAL COMPILATION
         {"#cond", {"", &Compiler::compile_gscond}},

goalc/compiler/compilation/CompilerControl.cpp

@@ -816,3 +816,8 @@ Val* Compiler::compile_gen_docs(const goos::Object& form, const goos::Object& re
 
   return get_none();
 }
+
+Val* Compiler::compile_gc_text(const goos::Object&, const goos::Object&, Env*) {
+  m_goos.reader.db.clear_info();
+  return get_none();
+}

goalc/compiler/compilation/Define.cpp

@@ -42,7 +42,11 @@ Val* Compiler::compile_define(const goos::Object& form, const goos::Object& rest
     // The third case - immediate lambdas - don't get passed to a define,
     //   so this won't cause those to live for longer than they should
     if ((as_lambda->func && as_lambda->func->settings.allow_inline) || !as_lambda->func) {
-      m_inlineable_functions[sym.as_symbol()] = as_lambda;
+      auto& f = m_inlineable_functions[sym.as_symbol()];
+      // default inline if we have to (because no code), or if that's the option.
+      f.inline_by_default = (!as_lambda->func) || as_lambda->func->settings.inline_by_default;
+      f.lambda = as_lambda->lambda;
+      f.type = as_lambda->type();
     }
     // Most defines come via macro invokations, we want the TRUE defining form location
     // if we can get it

goalc/compiler/compilation/Function.cpp

@@ -33,8 +33,7 @@ const goos::Object& get_lambda_body(const goos::Object& def) {
  * when used in a function call. This only works for immediaate function calls, you can't "save"
  * an (inline my-func) into a function pointer.
  *
- * If inlining is not possible (function disallows inlining or didn't save its code), throw an
- * error.
+ * If inlining is not possible (function didn't save its code), throw an error.
  */
 Val* Compiler::compile_inline(const goos::Object& form, const goos::Object& rest, Env* env) {
   (void)env;
@@ -47,12 +46,8 @@ Val* Compiler::compile_inline(const goos::Object& form, const goos::Object& rest
                          args.unnamed.at(0).print());
   }
 
-  if (kv->second->func && !kv->second->func->settings.allow_inline) {
-    throw_compiler_error(form,
-                         "Cannot inline {} because inlining of this function was disallowed.");
-  }
   auto fe = env->function_env();
-  return fe->alloc_val<InlinedLambdaVal>(kv->second->type(), kv->second);
+  return fe->alloc_val<InlinedLambdaVal>(kv->second.type, kv->second);
 }
 
 Val* Compiler::compile_local_vars(const goos::Object& form, const goos::Object& rest, Env* env) {
@@ -333,13 +328,11 @@ Val* Compiler::compile_function_or_method_call(const goos::Object& form, Env* en
     auto kv = m_inlineable_functions.find(uneval_head.as_symbol());
     if (kv != m_inlineable_functions.end()) {
       // it's inlinable.  However, we do not always inline an inlinable function by default
-      if (kv->second->func ==
-              nullptr ||  // only-inline, we must inline it as there is no code generated for it
-          kv->second->func->settings
-              .inline_by_default) {  // inline when possible, so we should inline
-
+      if (kv->second.inline_by_default) {  // inline when possible, so we should inline
         auto_inline = true;
-        head = kv->second;
+        auto* lv = env->function_env()->alloc_val<LambdaVal>(kv->second.type);
+        lv->lambda = kv->second.lambda;
+        head = lv;
       }
     }
   }
@@ -394,7 +387,8 @@ Val* Compiler::compile_function_or_method_call(const goos::Object& form, Env* en
       auto head_as_inlined_lambda = dynamic_cast<InlinedLambdaVal*>(head);
       if (head_as_inlined_lambda) {
         // yes, remember the lambda that contains and flag that we're inlining.
-        head_as_lambda = head_as_inlined_lambda->lv;
+        head_as_lambda = env->function_env()->alloc_val<LambdaVal>(head_as_inlined_lambda->lv.type);
+        head_as_lambda->lambda = head_as_inlined_lambda->lv.lambda;
         got_inlined_lambda = true;
       }
     }

goalc/debugger/DebugInfo.cpp

@@ -9,8 +9,9 @@ DebugInfo::DebugInfo(std::string obj_name) : m_obj_name(std::move(obj_name)) {}
 std::string FunctionDebugInfo::disassemble_debug_info(bool* had_failure,
                                                       const goos::Reader* reader) {
   std::string result = fmt::format("[{}]\n", name);
-  result += disassemble_x86_function(generated_code.data(), generated_code.size(), reader, 0x10000,
-                                     0x10000, instructions, function.get(), had_failure, true);
+  result +=
+      disassemble_x86_function(generated_code.data(), generated_code.size(), reader, 0x10000,
+                               0x10000, instructions, code_sources, ir_strings, had_failure, true);
 
   return result;
 }

goalc/debugger/DebugInfo.h

@@ -14,6 +14,11 @@
 
 class FunctionEnv;
 
+namespace goos {
+class Object;
+class HeapObject;
+}  // namespace goos
+
 /*!
  * FunctionDebugInfo stores per-function debugging information.
  * For now, it is pretty basic, but it will eventually contain stuff like stack frame info
@@ -26,9 +31,11 @@ struct FunctionDebugInfo {
   std::string name;
   std::string obj_name;
 
-  std::shared_ptr<FunctionEnv> function;
   std::vector<InstructionInfo> instructions;  // contains mapping to IRs
 
+  std::vector<std::shared_ptr<goos::HeapObject>> code_sources;
+  std::vector<std::string> ir_strings;
+
   // the actual bytes in the object file.
   std::vector<u8> generated_code;
   std::optional<int> stack_usage;

goalc/debugger/Debugger.cpp

@@ -400,8 +400,8 @@ Disassembly Debugger::disassemble_at_rip(const InstructionPointerInfo& info) {
       result.text += disassemble_x86_function(
           function_mem.data(), function_mem.size(), m_reader,
           m_debug_context.base + info.map_entry->start_addr + func_info->offset_in_seg,
-          rip + rip_offset, func_info->instructions, func_info->function.get(), &result.failed,
-          false);
+          rip + rip_offset, func_info->instructions, func_info->code_sources, func_info->ir_strings,
+          &result.failed, false);
     }
   } else {
     result.failed = true;

goalc/debugger/disassemble.cpp

@@ -77,15 +77,17 @@ std::string disassemble_x86(u8* data, int len, u64 base_addr, u64 highlight_addr
 static constexpr int FORM_DUMP_SIZE_REV = 4;
 static constexpr int FORM_DUMP_SIZE_FWD = 4;
 
-std::string disassemble_x86_function(u8* data,
-                                     int len,
-                                     const goos::Reader* reader,
-                                     u64 base_addr,
-                                     u64 highlight_addr,
-                                     const std::vector<InstructionInfo>& x86_instructions,
-                                     const FunctionEnv* fenv,
-                                     bool* had_failure,
-                                     bool print_whole_function) {
+std::string disassemble_x86_function(
+    u8* data,
+    int len,
+    const goos::Reader* reader,
+    u64 base_addr,
+    u64 highlight_addr,
+    const std::vector<InstructionInfo>& x86_instructions,
+    const std::vector<std::shared_ptr<goos::HeapObject>>& code_sources,
+    const std::vector<std::string>& ir_strings,
+    bool* had_failure,
+    bool print_whole_function) {
   std::string result;
   ZydisDecoder decoder;
   ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, ZYDIS_STACK_WIDTH_64);
@@ -94,8 +96,6 @@ std::string disassemble_x86_function(u8* data,
   ZydisDecodedInstruction instr;
   ZydisDecodedOperand op[ZYDIS_MAX_OPERAND_COUNT_VISIBLE];
 
-  const auto& irs = fenv->code();
-
   constexpr int print_buff_size = 512;
   char print_buff[print_buff_size];
   int offset = 0;
@@ -155,8 +155,8 @@ std::string disassemble_x86_function(u8* data,
 
       std::string line;
 
-      if (current_ir_idx >= 0 && current_ir_idx < int(irs.size())) {
-        auto source = reader->db.try_get_short_info(fenv->code_source().at(current_ir_idx));
+      if (current_ir_idx >= 0 && current_ir_idx < int(ir_strings.size())) {
+        auto source = reader->db.try_get_short_info(code_sources.at(current_ir_idx));
         if (source) {
           if (source->filename != current_filename ||
               source->line_idx_to_display != current_file_line ||
@@ -187,12 +187,12 @@ std::string disassemble_x86_function(u8* data,
                                       print_buff, print_buff_size, base_addr);
       line += print_buff;
 
-      if (print_ir && current_ir_idx >= 0 && current_ir_idx < int(irs.size())) {
+      if (print_ir && current_ir_idx >= 0 && current_ir_idx < int(ir_strings.size())) {
         if (line.size() < 50) {
           line.append(50 - line.size(), ' ');
         }
         line += " ";
-        line += irs.at(current_ir_idx)->print();
+        line += ir_strings.at(current_ir_idx);
       }
 
       if (warn_messed_up) {

goalc/debugger/disassemble.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <memory>
 #include <string>
 #include <vector>
 
@@ -11,7 +12,9 @@ class FunctionEnv;
 
 namespace goos {
 class Reader;
-}
+class Object;
+class HeapObject;
+}  // namespace goos
 
 struct InstructionInfo {
   emitter::Instruction instruction;  //! the actual x86 instruction
@@ -29,12 +32,14 @@ struct InstructionInfo {
 std::string disassemble_x86(u8* data, int len, u64 base_addr);
 std::string disassemble_x86(u8* data, int len, u64 base_addr, u64 highlight_addr);
 
-std::string disassemble_x86_function(u8* data,
-                                     int len,
-                                     const goos::Reader* reader,
-                                     u64 base_addr,
-                                     u64 highlight_addr,
-                                     const std::vector<InstructionInfo>& x86_instructions,
-                                     const FunctionEnv* fenv,
-                                     bool* had_failure,
-                                     bool print_whole_function);
+std::string disassemble_x86_function(
+    u8* data,
+    int len,
+    const goos::Reader* reader,
+    u64 base_addr,
+    u64 highlight_addr,
+    const std::vector<InstructionInfo>& x86_instructions,
+    const std::vector<std::shared_ptr<goos::HeapObject>>& code_sources,
+    const std::vector<std::string>& ir_strings,
+    bool* had_failure,
+    bool print_whole_function);

goalc/emitter/IGen.h

@@ -2082,7 +2082,7 @@ class IGen {
    */
   static Instruction null() {
     Instruction i(0);
-    i.is_null = true;
+    i.m_flags |= Instruction::kIsNull;
     return i;
   }
 

goalc/emitter/Instruction.h

@@ -140,46 +140,46 @@ struct Instruction {
   Instruction(uint8_t opcode) : op(opcode) {}
   uint8_t op;
 
-  bool op2_set = false;
+  enum Flags {
+    kOp2Set = (1 << 0),
+    kOp3Set = (1 << 1),
+    kIsNull = (1 << 2),
+    kSetRex = (1 << 3),
+    kSetModrm = (1 << 4),
+    kSetSib = (1 << 5),
+    kSetDispImm = (1 << 6),
+    kSetImm = (1 << 7),
+  };
+
+  u8 m_flags = 0;
+
   uint8_t op2;
 
-  bool op3_set = false;
   uint8_t op3;
 
-  // if true, don't emit anything
-  bool is_null = false;
-
-  // flag to indicate it's the first instruction of a function and needs align and type tag
-  bool is_function_start = false;
-
-  int n_vex = 0;
+  u8 n_vex = 0;
   uint8_t vex[3] = {0, 0, 0};
 
   // the rex byte
-  bool set_rex = false;
   uint8_t m_rex = 0;
 
   // the modrm byte
-  bool set_modrm = false;
   uint8_t m_modrm = 0;
 
   // the sib byte
-  bool set_sib = false;
   uint8_t m_sib = 0;
 
   // the displacement
-  bool set_disp_imm = false;
   Imm disp;
 
   // the immediate
-  bool set_imm = false;
   Imm imm;
 
   /*!
    * Move opcode byte 0 to before the rex prefix.
    */
   void swap_op0_rex() {
-    if (!set_rex)
+    if (!(m_flags & kSetRex))
       return;
     auto temp = op;
     op = m_rex;
@@ -188,17 +188,17 @@ struct Instruction {
 
   void set(REX r) {
     m_rex = r();
-    set_rex = true;
+    m_flags |= kSetRex;
   }
 
   void set(ModRM modrm) {
     m_modrm = modrm();
-    set_modrm = true;
+    m_flags |= kSetModrm;
   }
 
   void set(SIB sib) {
     m_sib = sib();
-    set_sib = true;
+    m_flags |= kSetSib;
   }
 
   void set(VEX3 vex3) {
@@ -217,26 +217,26 @@ struct Instruction {
 
   void set_disp(Imm i) {
     disp = i;
-    set_disp_imm = true;
+    m_flags |= kSetDispImm;
   }
 
   void set(Imm i) {
     imm = i;
-    set_imm = true;
+    m_flags |= kSetImm;
   }
 
   void set_op2(uint8_t b) {
-    op2_set = true;
+    m_flags |= kOp2Set;
     op2 = b;
   }
 
   void set_op3(uint8_t b) {
-    op3_set = true;
+    m_flags |= kOp3Set;
     op3 = b;
   }
 
   int get_imm_size() const {
-    if (set_imm) {
+    if (m_flags & kSetImm) {
       return imm.size;
     } else {
       return 0;
@@ -244,7 +244,7 @@ struct Instruction {
   }
 
   int get_disp_size() const {
-    if (set_disp_imm) {
+    if (m_flags & kSetDispImm) {
       return disp.size;
     } else {
       return 0;
@@ -806,7 +806,7 @@ struct Instruction {
   }
 
   void add_rex() {
-    if (!set_rex) {
+    if (!(m_flags & kSetRex)) {
       set(REX());
     }
   }
@@ -880,21 +880,21 @@ struct Instruction {
    * Get the position of the disp immediate relative to the start of the instruction
    */
   int offset_of_disp() const {
-    if (is_null)
+    if (m_flags & kIsNull)
       return 0;
-    ASSERT(set_disp_imm);
+    ASSERT(m_flags & kSetDispImm);
     int offset = 0;
     offset += n_vex;
-    if (set_rex)
+    if (m_flags & kSetRex)
       offset++;
     offset++;  // opcode
-    if (op2_set)
+    if (m_flags & kOp2Set)
       offset++;
-    if (op3_set)
+    if (m_flags & kOp3Set)
       offset++;
-    if (set_modrm)
+    if (m_flags & kSetModrm)
       offset++;
-    if (set_sib)
+    if (m_flags & kSetSib)
       offset++;
     return offset;
   }
@@ -903,23 +903,23 @@ struct Instruction {
    * Get the position of the imm immediate relative to the start of the instruction
    */
   int offset_of_imm() const {
-    if (is_null)
+    if (m_flags & kIsNull)
       return 0;
-    ASSERT(set_imm);
+    ASSERT(m_flags & kSetImm);
     int offset = 0;
     offset += n_vex;
-    if (set_rex)
+    if (m_flags & kSetRex)
       offset++;
     offset++;  // opcode
-    if (op2_set)
+    if (m_flags & kOp2Set)
       offset++;
-    if (op3_set)
+    if (m_flags & kOp3Set)
       offset++;
-    if (set_modrm)
+    if (m_flags & kSetModrm)
       offset++;
-    if (set_sib)
+    if (m_flags & kSetSib)
       offset++;
-    if (set_disp_imm)
+    if (m_flags & kSetDispImm)
       offset += disp.size;
     return offset;
   }
@@ -928,7 +928,7 @@ struct Instruction {
    * Emit into a buffer and return how many bytes written (can be zero)
    */
   uint8_t emit(uint8_t* buffer) const {
-    if (is_null)
+    if (m_flags & kIsNull)
       return 0;
     uint8_t count = 0;
 
@@ -936,35 +936,35 @@ struct Instruction {
       buffer[count++] = vex[i];
     }
 
-    if (set_rex) {
+    if (m_flags & kSetRex) {
       buffer[count++] = m_rex;
     }
 
     buffer[count++] = op;
 
-    if (op2_set) {
+    if (m_flags & kOp2Set) {
       buffer[count++] = op2;
     }
 
-    if (op3_set) {
+    if (m_flags & kOp3Set) {
       buffer[count++] = op3;
     }
 
-    if (set_modrm) {
+    if (m_flags & kSetModrm) {
       buffer[count++] = m_modrm;
     }
 
-    if (set_sib) {
+    if (m_flags & kSetSib) {
       buffer[count++] = m_sib;
     }
 
-    if (set_disp_imm) {
+    if (m_flags & kSetDispImm) {
       for (int i = 0; i < disp.size; i++) {
         buffer[count++] = disp.v_arr[i];
       }
     }
 
-    if (set_imm) {
+    if (m_flags & kSetImm) {
       for (int i = 0; i < imm.size; i++) {
         buffer[count++] = imm.v_arr[i];
       }
@@ -973,41 +973,41 @@ struct Instruction {
   }
 
   uint8_t length() const {
-    if (is_null)
+    if (m_flags & kIsNull)
       return 0;
     uint8_t count = 0;
 
     count += n_vex;
 
-    if (set_rex) {
+    if (m_flags & kSetRex) {
       count++;
     }
 
     count++;
 
-    if (op2_set) {
+    if (m_flags & kOp2Set) {
       count++;
     }
 
-    if (op3_set) {
+    if (m_flags & kOp3Set) {
       count++;
     }
 
-    if (set_modrm) {
+    if (m_flags & kSetModrm) {
       count++;
     }
 
-    if (set_sib) {
+    if (m_flags & kSetSib) {
       count++;
     }
 
-    if (set_disp_imm) {
+    if (m_flags & kSetDispImm) {
       for (int i = 0; i < disp.size; i++) {
         count++;
       }
     }
 
-    if (set_imm) {
+    if (m_flags & kSetImm) {
       for (int i = 0; i < imm.size; i++) {
         count++;
       }

goalc/regalloc/allocator_interface.h

@@ -82,9 +82,9 @@ struct StackOp {
  * For a single IR Instruction.
  */
 struct Assignment {
-  enum class Kind { STACK, REGISTER, UNASSIGNED } kind = Kind::UNASSIGNED;
+  int stack_slot = -1;  //! index of the slot, if we are ever spilled
+  enum class Kind : u8 { STACK, REGISTER, UNASSIGNED } kind = Kind::UNASSIGNED;
   emitter::Register reg = -1;  //! where the IRegister is now
-  int stack_slot = -1;         //! index of the slot, if we are ever spilled
   bool spilled = false;        //! are we ever spilled
 
   std::string to_string() const;