Make funcfiftlib compilation compatible with modern compilers (#618)

* wip: make funcfiftlib compilation compatible with modern compilers

* wip: add methods needed for another compiler

* fix: tdutils port config if emscripten

* feat: func source and realpath callback

* fix: invalid fift compilation exceptions

---------

Co-authored-by: krigga <krigga7@gmail.com>

crypto/CMakeLists.txt

@@ -300,6 +300,10 @@ target_link_libraries(test-ed25519-crypto PUBLIC ton_crypto)
 add_library(fift-lib ${FIFT_SOURCE})
 target_include_directories(fift-lib PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
 target_link_libraries(fift-lib PUBLIC ton_crypto ton_db tdutils ton_block)
+if (USE_EMSCRIPTEN)
+  target_link_options(fift-lib PRIVATE -fexceptions)
+  target_compile_options(fift-lib PRIVATE -fexceptions)
+endif()
 set_target_properties(fift-lib PROPERTIES OUTPUT_NAME fift)
 
 add_executable(fift fift/fift-main.cpp)
@@ -328,17 +332,20 @@ if (USE_EMSCRIPTEN)
   add_executable(funcfiftlib funcfiftlib/funcfiftlib.cpp ${FUNC_LIB_SOURCE})
   target_include_directories(funcfiftlib PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
   target_link_libraries(funcfiftlib PUBLIC fift-lib src_parser git)
-  target_link_options(funcfiftlib PRIVATE -sEXPORTED_RUNTIME_METHODS=FS,ccall,cwrap,_malloc,free,UTF8ToString,stringToUTF8)
-  target_link_options(funcfiftlib PRIVATE -sEXPORTED_FUNCTIONS=_func_compile,_version)
+  target_link_options(funcfiftlib PRIVATE -sEXPORTED_RUNTIME_METHODS=FS,ccall,cwrap,UTF8ToString,stringToUTF8,lengthBytesUTF8,addFunction,removeFunction,setValue)
+  target_link_options(funcfiftlib PRIVATE -sEXPORTED_FUNCTIONS=_func_compile,_version,_malloc,_free,_setThrew)
   target_link_options(funcfiftlib PRIVATE -sEXPORT_NAME=CompilerModule)
   target_link_options(funcfiftlib PRIVATE -sERROR_ON_UNDEFINED_SYMBOLS=0)
-  target_link_options(funcfiftlib PRIVATE -sFILESYSTEM=1)
+  target_link_options(funcfiftlib PRIVATE -sFILESYSTEM=1 -lnodefs.js)
   target_link_options(funcfiftlib PRIVATE -Oz)
   target_link_options(funcfiftlib PRIVATE -sIGNORE_MISSING_MAIN=1)
   target_link_options(funcfiftlib PRIVATE -sAUTO_NATIVE_LIBRARIES=0)
   target_link_options(funcfiftlib PRIVATE -sMODULARIZE=1)
+  target_link_options(funcfiftlib PRIVATE -sALLOW_MEMORY_GROWTH=1)
+  target_link_options(funcfiftlib PRIVATE -sALLOW_TABLE_GROWTH=1)
   target_link_options(funcfiftlib PRIVATE --embed-file ${CMAKE_CURRENT_SOURCE_DIR}/fift/lib@/fiftlib)
-  target_compile_options(funcfiftlib PRIVATE -sDISABLE_EXCEPTION_CATCHING=0)
+  target_link_options(funcfiftlib PRIVATE -fexceptions)
+  target_compile_options(funcfiftlib PRIVATE -fexceptions)
 endif()
 
 add_executable(tlbc tl/tlbc.cpp)

crypto/func/func-main.cpp

@@ -123,5 +123,7 @@ int main(int argc, char* const argv[]) {
     sources.push_back(std::string(argv[optind++]));
   }
 
+  funC::read_callback = funC::fs_read_callback;
+
   return funC::func_proceed(sources, *outs, std::cerr);
 }

crypto/func/func.cpp

@@ -30,6 +30,8 @@
 #include "parser/lexer.h"
 #include <getopt.h>
 #include "git.h"
+#include <fstream>
+#include "td/utils/port/path.h"
 
 namespace funC {
 
@@ -39,6 +41,28 @@ bool interactive = false;
 GlobalPragma pragma_allow_post_modification{"allow-post-modification"};
 GlobalPragma pragma_compute_asm_ltr{"compute-asm-ltr"};
 std::string generated_from, boc_output_filename;
+ReadCallback::Callback read_callback;
+
+td::Result<std::string> fs_read_callback(ReadCallback::Kind kind, const char* query) {
+  switch (kind) {
+    case ReadCallback::Kind::ReadFile: {
+      std::ifstream ifs{query};
+      if (ifs.fail()) {
+        auto msg = std::string{"cannot open source file `"} + query + "`";
+        return td::Status::Error(msg);
+      }
+      std::stringstream ss;
+      ss << ifs.rdbuf();
+      return ss.str();
+    }
+    case ReadCallback::Kind::Realpath: {
+      return td::realpath(td::CSlice(query));
+    }
+    default: {
+      return td::Status::Error("Unknown query kind");
+    }
+  }
+}
 
 /*
  *

crypto/func/func.h

@@ -30,6 +30,7 @@
 #include "parser/srcread.h"
 #include "parser/lexer.h"
 #include "parser/symtable.h"
+#include "td/utils/Status.h"
 
 namespace funC {
 
@@ -845,6 +846,35 @@ extern std::vector<SymDef*> glob_func, glob_vars;
  * 
  */
 
+class ReadCallback {
+public:
+  /// Noncopyable.
+  ReadCallback(ReadCallback const&) = delete;
+  ReadCallback& operator=(ReadCallback const&) = delete;
+
+  enum class Kind
+  {
+    ReadFile,
+    Realpath
+  };
+
+  static std::string kindString(Kind _kind)
+  {
+    switch (_kind)
+    {
+    case Kind::ReadFile:
+      return "source";
+    case Kind::Realpath:
+      return "realpath";
+    default:
+      throw ""; // todo ?
+    }
+  }
+
+  /// File reading or generic query callback.
+  using Callback = std::function<td::Result<std::string>(ReadCallback::Kind, const char*)>;
+};
+
 // defined in parse-func.cpp
 bool parse_source(std::istream* is, const src::FileDescr* fdescr);
 bool parse_source_file(const char* filename, src::Lexem lex = {}, bool is_main = false);
@@ -1691,6 +1721,9 @@ void define_builtins();
 extern int verbosity, indent, opt_level;
 extern bool stack_layout_comments, op_rewrite_comments, program_envelope, asm_preamble, interactive;
 extern std::string generated_from, boc_output_filename;
+extern ReadCallback::Callback read_callback;
+
+td::Result<std::string> fs_read_callback(ReadCallback::Kind kind, const char* query);
 
 class GlobalPragma {
  public:

crypto/func/parse-func.cpp

@@ -22,8 +22,6 @@
 #include "openssl/digest.hpp"
 #include "block/block.h"
 #include "block-parse.h"
-#include <fstream>
-#include "td/utils/port/path.h"
 
 namespace sym {
 
@@ -1757,7 +1755,7 @@ bool parse_source_file(const char* filename, src::Lexem lex, bool is_main) {
     }
   }
 
-  auto path_res = td::realpath(td::CSlice(filename));
+  auto path_res = read_callback(ReadCallback::Kind::Realpath, filename);
   if (path_res.is_error()) {
     auto error = path_res.move_as_error();
     lex.error(error.message().c_str());
@@ -1784,17 +1782,19 @@ bool parse_source_file(const char* filename, src::Lexem lex, bool is_main) {
   source_files[real_filename] = cur_source;
   cur_source->is_main = is_main;
   source_fdescr.push_back(cur_source);
-  std::ifstream ifs{filename};
-  if (ifs.fail()) {
-    auto msg = std::string{"cannot open source file `"} + filename + "`";
+  auto file_res = read_callback(ReadCallback::Kind::ReadFile, filename);
+  if (file_res.is_error()) {
+    auto msg = file_res.move_as_error().message().str();
     if (lex.tp) {
       lex.error(msg);
     } else {
       throw src::Fatal{msg};
     }
   }
+  auto file_str = file_res.move_as_ok();
+  std::stringstream ss{file_str};
   inclusion_locations.push(lex.loc);
-  bool res = parse_source(&ifs, cur_source);
+  bool res = parse_source(&ss, cur_source);
   inclusion_locations.pop();
   return res;
 }

crypto/funcfiftlib/funcfiftlib.cpp

@@ -30,6 +30,7 @@
 #include "td/utils/JsonBuilder.h"
 #include "fift/utils.h"
 #include "td/utils/base64.h"
+#include "td/utils/Status.h"
 #include <sstream>
 #include <iomanip>
 
@@ -99,6 +100,40 @@ td::Result<std::string> compile_internal(char *config_json) {
   return result_json.string_builder().as_cslice().str();
 }
 
+/// Callback used to retrieve additional source files or data.
+///
+/// @param _kind The kind of callback (a string).
+/// @param _data The data for the callback (a string).
+/// @param o_contents A pointer to the contents of the file, if found. Allocated via malloc().
+/// @param o_error A pointer to an error message, if there is one. Allocated via malloc().
+///
+/// The callback implementor must use malloc() to allocate storage for
+/// contents or error. The callback implementor must use free() to free
+/// said storage after func_compile returns.
+///
+/// If the callback is not supported, *o_contents and *o_error must be set to NULL.
+typedef void (*CStyleReadFileCallback)(char const* _kind, char const* _data, char** o_contents, char** o_error);
+
+funC::ReadCallback::Callback wrapReadCallback(CStyleReadFileCallback _readCallback)
+{
+  funC::ReadCallback::Callback readCallback;
+  if (_readCallback) {
+    readCallback = [=](funC::ReadCallback::Kind _kind, char const* _data) -> td::Result<std::string> {
+      char* contents_c = nullptr;
+      char* error_c = nullptr;
+      _readCallback(funC::ReadCallback::kindString(_kind).data(), _data, &contents_c, &error_c);
+      if (!contents_c && !error_c) {
+        return td::Status::Error("Callback not supported");
+      }
+      if (contents_c) {
+        return contents_c;
+      }
+      return td::Status::Error(std::string(error_c));
+    };
+  }
+  return readCallback;
+}
+
 extern "C" {
 
 const char* version() {
@@ -111,7 +146,13 @@ const char* version() {
   return strdup(version_json.string_builder().as_cslice().c_str());
 }
 
-const char *func_compile(char *config_json) {
+const char *func_compile(char *config_json, CStyleReadFileCallback callback) {
+  if (callback) {
+    funC::read_callback = wrapReadCallback(callback);
+  } else {
+    funC::read_callback = funC::fs_read_callback;
+  }
+
   auto res = compile_internal(config_json);
 
   if (res.is_error()) {

tdutils/td/utils/port/config.h

@@ -28,35 +28,33 @@
   #define TD_PORT_POSIX 1
 #endif
 
-#if TD_LINUX || TD_ANDROID || TD_TIZEN
+#if TD_EMSCRIPTEN
+  #define TD_POLL_POLL 1
+#elif TD_LINUX || TD_ANDROID || TD_TIZEN
   #define TD_POLL_EPOLL 1
-  #define TD_EVENTFD_LINUX 1
 #elif TD_FREEBSD || TD_OPENBSD || TD_NETBSD
   #define TD_POLL_KQUEUE 1
-  #define TD_EVENTFD_BSD 1
 #elif TD_CYGWIN
   #define TD_POLL_SELECT 1
-  #define TD_EVENTFD_BSD 1
-#elif TD_EMSCRIPTEN
-  #define TD_POLL_POLL 1
-  // #define TD_EVENTFD_UNSUPPORTED 1
 #elif TD_DARWIN
   #define TD_POLL_KQUEUE 1
-  #define TD_EVENTFD_BSD 1
 #elif TD_WINDOWS
   #define TD_POLL_WINEVENT 1
-  #define TD_EVENTFD_WINDOWS 1
 #else
   #error "Poll's implementation is not defined"
 #endif
 
-#if TD_EMSCRIPTEN
-  // #define TD_THREAD_UNSUPPORTED 1
-  #define TD_POLL_EPOLL 1
-  #define TD_EVENTFD_UNSUPPORTED 0
-  #define TD_THREAD_PTHREAD 1
+#if TD_LINUX || TD_ANDROID || TD_TIZEN
   #define TD_EVENTFD_LINUX 1
-#elif TD_TIZEN || TD_LINUX || TD_DARWIN
+#elif TD_FREEBSD || TD_OPENBSD || TD_NETBSD || TD_CYGWIN || TD_DARWIN
+  #define TD_EVENTFD_BSD 1
+#elif TD_WINDOWS
+  #define TD_EVENTFD_WINDOWS 1
+#else
+  #error "eventfd's implementation is not defined"
+#endif
+
+#if TD_TIZEN || TD_LINUX || TD_DARWIN || TD_EMSCRIPTEN
   #define TD_THREAD_PTHREAD 1
 #else
   #define TD_THREAD_STL 1

tdutils/td/utils/port/platform.h

@@ -20,6 +20,11 @@
 
 // clang-format off
 
+/*** Determine emscripten ***/
+#if defined(__EMSCRIPTEN__)
+  #define TD_EMSCRIPTEN 1
+#endif
+
 /*** Platform macros ***/
 #if defined(_WIN32) || defined(_WINDOWS) // _WINDOWS is defined by CMake
   #if defined(__cplusplus_winrt)
@@ -63,10 +68,11 @@
   #define TD_NETBSD 1
 #elif defined(__CYGWIN__)
   #define TD_CYGWIN 1
-#elif defined(__EMSCRIPTEN__)
-  #define TD_EMSCRIPTEN 1
-#elif defined(__unix__)  // all unices not caught above
-  #warning "Probably unsupported Unix platform. Feel free to try to compile"
+#elif defined(__unix__) // all unices not caught above
+  // supress if emscripten
+  #if !TD_EMSCRIPTEN
+    #warning "Probably unsupported Unix platform. Feel free to try to compile"
+  #endif
   #define TD_CYGWIN 1
 #else
   #error "Probably unsupported platform. Feel free to remove the error and try to recompile"