Add Assignment 2 tests

Fixes #5

.gitmodules

@@ -1,3 +1,9 @@
 [submodule "external/Catch2"]
 	path = external/Catch2
 	url = https://github.com/catchorg/Catch2
+[submodule "external/optional"]
+	path = external/optional
+	url = https://github.com/TartanLlama/optional
+[submodule "external/cppast"]
+	path = external/cppast
+	url = https://github.com/foonathan/cppast

CMakeLists.txt

@@ -1,9 +1,20 @@
 cmake_minimum_required(VERSION 3.10.2)
 
+set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+# Disable test compilation for the optional tl library.
+set(OPTIONAL_ENABLE_TESTS OFF CACHE INTERNAL "Enable tl::optional tests")
 
 include_directories(includes)
 add_subdirectory(external/Catch2)
+add_subdirectory(external/cppast)
+add_subdirectory(external/optional)
 
 add_executable(asgn1 src/asgn1/main.cpp)
 target_link_libraries(asgn1 Catch2)
+
+add_executable(asgn2 src/asgn2/main.cpp)
+target_link_libraries(asgn2 Catch2 cppast optional)

README.md

@@ -5,6 +5,7 @@
 This project uses `cmake` to manage dependencies. Compiling the assignment testing binaries involves running a few commands. This produces an "out of source" build.
 
 ```sh
+$ git submodule update --init --recursive
 $ mkdir build
 $ cd build
 $ cmake ..

includes/internal/exists.hpp

@@ -0,0 +1,14 @@
+#ifndef __CSE3150_TESTING_EXISTS_H
+#define __CSE3150_TESTING_EXISTS_H
+
+#include <string>
+#include <sys/stat.h>
+
+// https://stackoverflow.com/a/12774387
+inline bool exists (const std::string& name) {
+  struct stat buffer;
+  const auto status = stat(name.c_str(), &buffer);
+  return status == 0;
+}
+
+#endif

includes/internal/simple_parse.hpp

@@ -0,0 +1,14 @@
+#ifndef __CSE3150_TESTING_PARSER14_H
+#define __CSE3150_TESTING_PARSER14_H
+
+#include <cppast/libclang_parser.hpp>
+
+std::unique_ptr<cppast::cpp_file> simple_parse(const std::string& path) {
+  cppast::libclang_compile_config config;
+  cppast::cpp_entity_index idx;
+  cppast::stderr_diagnostic_logger logger;
+  cppast::libclang_parser parser(type_safe::ref(logger));
+  return parser.parse(idx, path, config);
+}
+
+#endif

src/asgn2/buildlist.hpp

@@ -0,0 +1,81 @@
+#ifndef __CSE3150_TESTING_BUILDLIST_H
+#define __CSE3150_TESTING_BUILDLIST_H
+
+#include <algorithm>
+#include <cppast/cpp_function.hpp>
+#include <cppast/cpp_template.hpp>
+#include <cppast/cpp_type.hpp>
+#include <cppast/libclang_parser.hpp>
+#include <cppast/visitor.hpp>
+#include <optional.hpp>
+
+tl::optional<const cppast::cpp_entity&> find_build_list_fn(const cppast::cpp_file& file) {
+  tl::optional<const cppast::cpp_entity&> result = tl::nullopt;
+
+  cppast::visit(file, [&](const cppast::cpp_entity& e, cppast::visitor_info info) {
+    const auto is_function = e.kind() == cppast::cpp_entity_kind::function_t;
+    const auto correct_name = e.name() == "buildList";
+
+    if (is_function && correct_name) {
+      result = e;
+    }
+
+    // Stop visiting further nodes if we found the one we're looking for.
+    const auto should_keep_looking = result == tl::nullopt;
+    return should_keep_looking;
+  });
+
+  return result;
+}
+
+bool type_is_list_template(const cppast::cpp_type& type) {
+  const auto is_template_instantiation = type.kind() == cppast::cpp_type_kind::template_instantiation_t;
+
+  if (!is_template_instantiation) {
+    return false;
+  }
+
+  const auto& template_instantiation = static_cast<const cppast::cpp_template_instantiation_type&>(type);
+  const auto primary_template = template_instantiation.primary_template();
+
+  return
+    primary_template.name() == "list" ||
+    primary_template.name() == "std::list";
+}
+
+bool function_takes_list_reference(const cppast::cpp_function& fn) {
+  const auto list_param = std::find_if(
+    fn.parameters().begin(),
+    fn.parameters().end(),
+    [](const cppast::cpp_function_parameter& parameter)
+  {
+    const auto& type = parameter.type();
+    const auto is_reference = type.kind() == cppast::cpp_type_kind::reference_t;
+
+    if (!is_reference) {
+      return false;
+    }
+
+    const auto& reference = static_cast<const cppast::cpp_reference_type&>(type);
+    return type_is_list_template(reference.referee());
+  });
+
+  return list_param != fn.parameters().end();
+}
+
+bool function_returns_list_pointer(const cppast::cpp_function& fn) {
+  const cppast::cpp_type& return_type = fn.return_type();
+  const cppast::cpp_type_kind& kind = return_type.kind();
+  const auto is_pointer = kind == cppast::cpp_type_kind::pointer_t;
+
+  if (!is_pointer) {
+    return false;
+  }
+
+  const auto& pointer = static_cast<const cppast::cpp_pointer_type&>(return_type);
+  const auto& pointee = pointer.pointee();
+
+  return type_is_list_template(pointer.pointee());
+}
+
+#endif

src/asgn2/main.cpp

@@ -0,0 +1,36 @@
+#define CATCH_CONFIG_RUNNER
+#include <catch2/catch.hpp>
+#include <iomanip>
+#include <iostream>
+#include "tests.cpp"
+
+// There doesn't seem to be a way to get the total number of check/require
+// statements from a Catch2 session. We will fortunately have to update
+// this value manually as tests are added/removed.
+const unsigned int TOTAL_ASSERTIONS = 13;
+
+int main(int argc, char* argv[]) {
+  const int num_failed = Catch::Session().run(argc, argv);
+  const bool perfect = num_failed == 0;
+
+  if (perfect) {
+    std::cout
+      << "Congratulations, you finished this assignment with a 100%!" << std::endl
+      << "Looks like we need to make the next one harder. ;)" << std::endl
+      << std::endl;
+  }
+
+  std::cout
+    << "Passed "
+    << (TOTAL_ASSERTIONS - num_failed) << '/' << TOTAL_ASSERTIONS
+    << std::endl;
+
+  std::cout
+    << "Your score: "
+    << std::fixed
+    << std::setprecision(2)
+    << (1 - (static_cast<float>(num_failed) / TOTAL_ASSERTIONS)) * 100 << "%"
+    << std::endl;
+
+  return perfect ? 0 : 1;
+}

src/asgn2/question.hpp

@@ -0,0 +1,70 @@
+#ifndef __CSE3150_TESTING_ASGN2_QUESTION_H
+#define __CSE3150_TESTING_ASGN2_QUESTION_H
+
+#include <iostream>
+#include <system_error>
+#include <subprocess.hpp>
+#include <internal/explode.hpp>
+#include <internal/exists.hpp>
+#include <optional.hpp>
+
+struct Question {
+  std::string id;
+  std::string desc;
+
+  std::string directory() const {
+    return "./asgn2/" + this->id + "/";
+  }
+
+  std::string wlist_path() const {
+    return this->directory() + "WList.cpp";
+  }
+
+  std::string binary_path() const {
+    return this->directory() + "wlist";
+  }
+
+  tl::optional<std::vector<std::string>> exec(const std::string& in) const {
+    if (!exists(this->binary_path())) {
+      return tl::nullopt;
+    }
+
+    try {
+      subprocess::popen cmd(this->binary_path(), {});
+      cmd.stdin() << in << std::endl;
+      cmd.close();
+
+      std::stringstream s;
+      s << cmd.stdout().rdbuf();
+      std::string out = s.str();
+
+      return explode(out, '\n');
+    } catch (std::system_error) {
+      return tl::nullopt;
+    }
+  }
+
+  bool passes_valgrind(const std::string& in) const {
+    if (!exists(this->binary_path())) {
+      return false;
+    }
+
+    try {
+      subprocess::popen cmd("valgrind", {
+        "--error-exitcode=1",
+        "--leak-check=full",
+        this->binary_path()
+      });
+      cmd.stdin() << in << std::endl;
+      cmd.close();
+      int exit_status = cmd.wait();
+      return exit_status != 1;
+    } catch (std::system_error) {
+      return false;
+    }
+
+    return false;
+  }
+};
+
+#endif

src/asgn2/tests.cpp

@@ -0,0 +1,78 @@
+#include <iostream>
+#include <subprocess.hpp>
+#include <internal/exists.hpp>
+#include <internal/simple_parse.hpp>
+#include "buildlist.hpp"
+#include "question.hpp"
+
+TEST_CASE("Assignment 2") {
+  const auto question = GENERATE(
+    Question { .id = "q1", .desc = "Q1: References" },
+    Question { .id = "q2", .desc = "Q2: Pointers" },
+    Question { .id = "q3", .desc = "Q3: Exception" }
+  );
+
+  DYNAMIC_SECTION(question.desc) {
+    DYNAMIC_SECTION("Compile " + question.id) {
+      // I'm not sure if running make before running our tests is a good idea or
+      // not yet. We'll see what responses are like. -Brandon
+      subprocess::popen make("make", {"-C", question.directory()});
+      const auto make_exit_status = make.wait();
+      INFO(make.stdout().rdbuf());
+      CHECK(make_exit_status == 0);
+    }
+
+    DYNAMIC_SECTION("Test a simple phrase for " + question.id) {
+      INFO("Make sure that the output is dumped one word per line");
+      CHECK_THAT(
+        *question.exec("Vim is better than emacs").disjunction({}),
+        Catch::Equals(std::vector<std::string>({ "Vim", "is", "better", "than", "emacs" }))
+      );
+    }
+
+    DYNAMIC_SECTION("Check a simple phrase with valgrind") {
+      INFO("Check that valgrind passes on your input and returns with no memory errors.");
+      CHECK(question.passes_valgrind("Emacs is better than vim"));
+    }
+
+    DYNAMIC_SECTION("Check that buildList (in " + question.wlist_path() + ") meets question criteria.") {
+      if (!exists(question.wlist_path())) {
+        FAIL("File not found: " + question.wlist_path());
+      }
+
+      const auto w_list_file = simple_parse(question.wlist_path());
+      if (w_list_file == nullptr) {
+        FAIL("Failed to parse wList.cpp file. Check the file's syntax and make sure it compiles.");
+      }
+
+      const auto build_list = find_build_list_fn(*w_list_file);
+      if (!build_list.has_value()) {
+        FAIL("Make sure the \"buildList\" function exists in \"asgn2/q1/WList.cpp\"");
+      }
+
+      // Ew... casting. (The first-party example uses it too, so this may be
+        // required by the cppast API design.)
+      auto& build_list_fn = static_cast<const cppast::cpp_function&>(*build_list);
+
+      if (question.id == "q1" || question.id == "q3") {
+        INFO("\"buildList\" needs to take an \"std::list\" as a reference for q1.\"");
+        CHECK(function_takes_list_reference(build_list_fn));
+      }
+
+      if (question.id == "q2") {
+        INFO("\"buildList\" needs to return an \"std::list\" pointer.\"");
+        CHECK(function_returns_list_pointer(build_list_fn));
+      }
+    }
+
+    if (question.id == "q3") {
+      SECTION("Output should be different if non-alphabetic character is entered") {
+        INFO("Have buildList throw an exception, then output an error (instead of the user input). The non-alphabetic characters should not be printed back out.");
+        CHECK_THAT(
+          *question.exec("Vim = emacs").disjunction({}),
+          !Catch::Equals(std::vector<std::string>({ "Vim", "=", "emacs" }))
+        );
+      }
+    }
+  }
+}