From b887e35554a4d274cba73d50d0de21601b395e52 Mon Sep 17 00:00:00 2001
From: Ashik K <root@ashik.se>
Date: Wed, 8 Mar 2023 14:18:12 +0100
Subject: [PATCH] wip: fetcher and parser for Wikimedia museum data project

---
 CMakeLists.txt |  19 ++++++
 fetcher.cc     |  52 +++++++++++++++
 parser.cc      | 174 +++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 245 insertions(+)
 create mode 100644 CMakeLists.txt
 create mode 100644 fetcher.cc
 create mode 100644 parser.cc

diff --git a/CMakeLists.txt b/CMakeLists.txt
new file mode 100644
index 0000000..d60d910
--- /dev/null
+++ b/CMakeLists.txt
@@ -0,0 +1,19 @@
+cmake_minimum_required(VERSION 3.14)
+project(testCurlPP LANGUAGES CXX)
+set(CMAKE_INCLUDE_CURRENT_DIR ON)
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+include(FindPkgConfig)
+pkg_check_modules(CURLPP REQUIRED curlpp)
+
+add_executable(fetch_all
+  fetcher.cc
+)
+add_executable(parse_all
+  parser.cc
+)
+
+target_link_libraries(fetch_all
+    ${CURLPP_LDFLAGS}
+)
diff --git a/fetcher.cc b/fetcher.cc
new file mode 100644
index 0000000..d548170
--- /dev/null
+++ b/fetcher.cc
@@ -0,0 +1,52 @@
+#include <sstream>
+#include <fstream>
+
+#include <cstdlib>
+#include <cstring>
+
+#include <curlpp/cURLpp.hpp>
+#include <curlpp/Easy.hpp>
+#include <curlpp/Options.hpp>
+#include <curlpp/Exception.hpp>
+
+#define DEBUG_LEVEL -1
+
+int main(int argc, char *argv[])
+{
+  if (argc !=2 ) {
+    std::cout<<"Please run as "<<argv[0] <<" <api key>"<<std::endl;
+    exit(0);
+  }
+  char url[256], outfilename[64];
+  for (int i = 0; i<4; i++) {
+    snprintf(url, sizeof url, "https://api.dimu.org/api/solr/select?q=Kosta&wt=json&fq=(artifact.exhibitionUids:\"336485B3-1E4E-443C-8800-F93BDD12520A\")&start=%d&rows=100&api.key=%s", i*100, argv[1]);
+    snprintf(outfilename, sizeof outfilename, "data_%d.json", i);
+   
+   /*if (DEBUG_LEVEL > 0) */std::cout<<"url to fetch is "<<std::endl<<url<<std::endl;
+   std::cout<<"writing output to "<<outfilename<<std::endl;
+  
+  try {
+    curlpp::Cleanup cleaner;
+    curlpp::Easy request1;
+
+    // Setting the URL to retrive.
+    request1.setOpt(new curlpp::options::Url(url));
+
+    if (DEBUG_LEVEL > 0) std::cout << request1 << std::endl;
+
+    std::ofstream outfile;
+    outfile.open (outfilename);
+    outfile << curlpp::options::Url(url) << std::endl;
+    outfile.close();
+  }
+  catch ( curlpp::LogicError & e ) {
+    if (DEBUG_LEVEL > 0) std::cout << e.what() << std::endl;
+  }
+  catch ( curlpp::RuntimeError & e ) {
+    if (DEBUG_LEVEL > 0) std::cout << e.what() << std::endl;
+  }
+
+ 
+}
+   return 0;
+}
diff --git a/parser.cc b/parser.cc
new file mode 100644
index 0000000..b5dbd61
--- /dev/null
+++ b/parser.cc
@@ -0,0 +1,174 @@
+#include <rapidjson/document.h>
+#include <rapidjson/filereadstream.h>
+
+#include <algorithm>
+#include <cstdio>
+#include <fstream>
+#include <iostream>
+#include <map>
+
+std::string yearstr(int year) {
+  return year==-1? "Unknown": std::to_string(year);
+}
+
+int main()
+{
+  char filename[64];
+  snprintf(filename, sizeof filename, "out.csv");
+  std::ofstream out_file1;
+  out_file1.open(filename);
+  out_file1<<"id, Caption/title, production start year, end year, Description, Source, image_filename, collection name, museum name, linceses\n";
+  for (auto i = 0; i<4; i++) {
+    char in_file_i[64];
+    snprintf(in_file_i, sizeof in_file_i, "data_%d.json", i);
+    // Open the file
+    FILE* fp = fopen(in_file_i, "rb");
+    // Check if the file was opened successfully
+    if (!fp) {
+      std::cerr << "Error: unable to open file"
+                << std::endl;
+      return -1;
+    }
+
+    // Read the file 
+    char readBuffer[192000];
+    rapidjson::FileReadStream is(fp, readBuffer, sizeof(readBuffer));
+
+    // Parse the JSON document
+    rapidjson::Document doc;
+    doc.ParseStream(is);
+
+    // Check if the document is valid
+    if (doc.HasParseError()) {
+      std::cerr << "Error: failed to parse JSON document"
+                << std::endl;
+      fclose(fp);
+      return 1;
+    }
+
+    // Close the file
+    fclose(fp);
+
+    if (doc.HasMember("response")
+        && doc["response"].IsObject()) {
+      const rapidjson::Value& response = doc["response"];
+      if (response.HasMember("docs")
+          && response["docs"].IsArray()) {
+        std::cout<<"yes, response is an array"<<std::endl;
+        for (rapidjson::SizeType i = 0; i < response["docs"].Size(); i++) {
+          //std::cout<<i<<std::endl;
+          int yearb = -1, yeare = -1;
+          std::string title = "";
+          const rapidjson::Value& article = response["docs"][i];
+
+          std::string article_id = "";
+          if (article.HasMember("identifier.id")) {
+            article_id = article["identifier.id"].GetString();
+          }
+
+          if (article.HasMember("artifact.ingress.title")) {
+            title  = article["artifact.ingress.title"].GetString();
+          }
+
+          if (article.HasMember("artifact.ingress.production.fromYear")) {
+            const rapidjson::Value& yearbval = article["artifact.ingress.production.fromYear"];
+            yearb = yearbval.GetInt();
+          }
+
+          if (article.HasMember("artifact.ingress.production.toYear")) {
+            const rapidjson::Value& yeareval = article["artifact.ingress.production.toYear"];
+            yeare = yeareval.GetInt();
+          }
+
+        std::string mediaid = "";
+        int picid = -1;
+        std::string picdim = "";
+        
+        if (article.HasMember("artifact.hasPictures") && article["artifact.hasPictures"].GetBool() == true) {
+
+          if (article.HasMember("artifact.defaultMediaIdentifier")) {
+            const rapidjson::Value& mediaidv = article["artifact.defaultMediaIdentifier"];
+            if (mediaidv.IsString()) {
+              mediaid = mediaidv.GetString();
+            }
+          }
+
+          if (article.HasMember("artifact.defaultPictureIndex") && article["artifact.defaultPictureIndex"].IsInt()) {
+            picid = article["artifact.defaultPictureIndex"].GetInt();
+          }
+
+          if (article.HasMember("artifact.defaultPictureDimension") && article["artifact.defaultPictureDimension"].IsString()) {
+            picdim = article["artifact.defaultPictureDimension"].GetString();
+          }
+        }
+        
+        //  image file is at https://mm.dimu.org/image/<identifier>?dimension=<dim>
+        // as per documentation available at http://api.dimu.org/doc/public_api.html
+        char imglink[128];
+        snprintf(imglink, sizeof imglink, "https://mm.dimu.org/image/%s", mediaid.c_str());
+
+        
+        
+        char descfilename[64];
+        snprintf(descfilename, sizeof descfilename, "%s.txt", article_id.c_str());
+
+        std::string description = "";
+        if (article.HasMember("artifact.ingress.description")) {
+          const rapidjson::Value& descv = article["artifact.ingress.description"];
+          if (descv.IsString()) {
+            description = descv.GetString();
+            std::replace( description.begin(), description.end(), ',', ':'); 
+            std::replace( description.begin(), description.end(), '\n', ' ');
+            std::ofstream desc_file;
+            desc_file.open(descfilename);
+            desc_file<<description;
+            desc_file.close();
+          }
+        }
+
+        std::string license = "";
+        if (article.HasMember("artifact.ingress.license")) {
+          if (article[ "artifact.ingress.license"].IsString())
+            license = article[ "artifact.ingress.license"].GetString();
+          else if (article[ "artifact.ingress.license"].IsArray()) {
+            for (auto i = 0; i<article[ "artifact.ingress.license"].Size(); i++) {
+              license += article[ "artifact.ingress.license"][i].GetString();
+            }
+          }
+        }
+
+        bool is_cc_license = false;
+        if (license.find("CC by")!= std::string::npos) {
+          is_cc_license = true;
+          char imgfetch[256];
+          snprintf(imgfetch, sizeof imgfetch, "wget %s -O \"%s-%s.jpeg\"", imglink, article_id.c_str(), mediaid.c_str());
+          std::cout<<imgfetch<<std::endl;
+          //          system (imgfetch);
+          std::cout<<"Found CC by license, fetched the image for "<< mediaid<<"\n";
+        }
+        if (is_cc_license) {
+        out_file1<<
+          /*  article["artifact.defaultMediaIdentifier"].GetString()<<
+              ", "<< article["artifact.defaultPictureIndex"].GetInt()<<
+              ", "<< article["artifact.defaultPictureDimension"].GetString()<<
+              ", "<<*/
+          article_id <<
+          ", "<< title <<
+          ", "<< yearstr(yearb) <<
+          ", "<< yearstr(yeare) <<
+          ", "<< descfilename <<
+          ", "<< imglink <<
+          ", "<< article_id+"-"+ mediaid +".jpeg" <<
+          ", "<<"Länge leve Kosta! exhibition" <<
+          ", "<<"Kulturparken Småland / Smålands museum" <<
+          ", "<< license <<
+          std::endl;
+        }
+        }
+      }
+    }
+  }
+  out_file1.close();   
+  return 0;
+}
+