305 lines
11 KiB
C++
305 lines
11 KiB
C++
#include <rapidjson/document.h>
|
|
#include <rapidjson/filereadstream.h>
|
|
|
|
#include <cstdlib>
|
|
#include <unistd.h>
|
|
|
|
#include <algorithm>
|
|
#include <cstdio>
|
|
#include <fstream>
|
|
#include <iostream>
|
|
#include <map>
|
|
|
|
#define debug_level 0
|
|
std::string yearstr(int year) {
|
|
return year == -1 ? "Unknown" : std::to_string(year);
|
|
}
|
|
|
|
int main(int argc, char **argv) {
|
|
int c;
|
|
bool download_mode = false;
|
|
int numfiles = 0;
|
|
while ((c = getopt(argc, argv, "d:n:")) != -1)
|
|
switch (c) {
|
|
case 'd': // download mode
|
|
download_mode = true;
|
|
break;
|
|
case 'n': // number of files to process
|
|
numfiles = atoi(optarg);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
char filename[64];
|
|
snprintf(filename, sizeof filename, "out.csv");
|
|
std::ofstream out_file1;
|
|
out_file1.open(filename);
|
|
out_file1 << "id, Caption/title, production start year, end year, "
|
|
"Description, Item url, Image Source, image_filename, subjects, "
|
|
"date published, collection name, museum name, exif_model, "
|
|
"exif_iso, exif_focallength, exif_exposuretime, exif_aperture, "
|
|
"exif_datetimeoriginal, liceses\n";
|
|
for (auto i = 0; i < numfiles; i++) {
|
|
char in_file_i[64];
|
|
snprintf(in_file_i, sizeof in_file_i, "data_%d.json", i);
|
|
// Open the file
|
|
FILE *fp = fopen(in_file_i, "rb");
|
|
// Check if the file was opened successfully
|
|
if (!fp) {
|
|
std::cerr << "Error: unable to open file" << std::endl;
|
|
return -1;
|
|
}
|
|
|
|
// Read the file
|
|
char readBuffer[192000];
|
|
rapidjson::FileReadStream is(fp, readBuffer, sizeof(readBuffer));
|
|
|
|
// Parse the JSON document
|
|
rapidjson::Document doc;
|
|
doc.ParseStream(is);
|
|
|
|
// Check if the document is valid
|
|
if (doc.HasParseError()) {
|
|
std::cerr << "Error: failed to parse JSON document" << std::endl;
|
|
fclose(fp);
|
|
return 1;
|
|
}
|
|
|
|
// Close the file
|
|
fclose(fp);
|
|
|
|
if (doc.HasMember("response") && doc["response"].IsObject()) {
|
|
const rapidjson::Value &response = doc["response"];
|
|
if (response.HasMember("docs") && response["docs"].IsArray()) {
|
|
for (rapidjson::SizeType i = 0; i < response["docs"].Size(); i++) {
|
|
// std::cout<<i<<std::endl;
|
|
int yearb = -1, yeare = -1;
|
|
std::string title = "";
|
|
const rapidjson::Value &article = response["docs"][i];
|
|
|
|
std::string article_id = "";
|
|
if (article.HasMember("identifier.id")) {
|
|
article_id = article["identifier.id"].GetString();
|
|
}
|
|
|
|
if (article.HasMember("artifact.ingress.title")) {
|
|
title = article["artifact.ingress.title"].GetString();
|
|
}
|
|
|
|
if (article.HasMember("artifact.ingress.production.fromYear")) {
|
|
const rapidjson::Value &yearbval =
|
|
article["artifact.ingress.production.fromYear"];
|
|
yearb = yearbval.GetInt();
|
|
}
|
|
|
|
if (article.HasMember("artifact.ingress.production.toYear")) {
|
|
const rapidjson::Value &yeareval =
|
|
article["artifact.ingress.production.toYear"];
|
|
yeare = yeareval.GetInt();
|
|
}
|
|
|
|
std::string unique_id = "";
|
|
if (article.HasMember("artifact.uniqueId")) {
|
|
const rapidjson::Value &uniqidval = article["artifact.uniqueId"];
|
|
if (uniqidval.IsString()) {
|
|
unique_id = uniqidval.GetString();
|
|
}
|
|
}
|
|
|
|
std::string mediaid = "";
|
|
int picid = -1;
|
|
std::string picdim = "";
|
|
|
|
if (article.HasMember("artifact.hasPictures") &&
|
|
article["artifact.hasPictures"].GetBool() == true) {
|
|
|
|
if (article.HasMember("artifact.defaultMediaIdentifier")) {
|
|
const rapidjson::Value &mediaidv =
|
|
article["artifact.defaultMediaIdentifier"];
|
|
if (mediaidv.IsString()) {
|
|
mediaid = mediaidv.GetString();
|
|
}
|
|
}
|
|
|
|
if (article.HasMember("artifact.defaultPictureIndex") &&
|
|
article["artifact.defaultPictureIndex"].IsInt()) {
|
|
picid = article["artifact.defaultPictureIndex"].GetInt();
|
|
}
|
|
|
|
if (article.HasMember("artifact.defaultPictureDimension") &&
|
|
article["artifact.defaultPictureDimension"].IsString()) {
|
|
picdim = article["artifact.defaultPictureDimension"].GetString();
|
|
}
|
|
}
|
|
|
|
// image file is at
|
|
// https://mm.dimu.org/image/<identifier>?dimension=<dim>
|
|
// as per documentation available at
|
|
// http://api.dimu.org/doc/public_api.html
|
|
char imglink[128];
|
|
snprintf(imglink, sizeof imglink, "https://mm.dimu.org/image/%s",
|
|
mediaid.c_str());
|
|
|
|
char itemlink[128];
|
|
snprintf(itemlink, sizeof itemlink, "https://digitaltmuseum.se/%s",
|
|
unique_id.c_str());
|
|
|
|
char descfilename[64];
|
|
snprintf(descfilename, sizeof descfilename, "%s.txt",
|
|
article_id.c_str());
|
|
|
|
std::string description = "";
|
|
if (article.HasMember("artifact.ingress.description")) {
|
|
const rapidjson::Value &descv =
|
|
article["artifact.ingress.description"];
|
|
if (descv.IsString()) {
|
|
description = descv.GetString();
|
|
std::replace(description.begin(), description.end(), ',', ':');
|
|
std::replace(description.begin(), description.end(), '\n', ' ');
|
|
std::ofstream desc_file;
|
|
desc_file.open(descfilename);
|
|
desc_file << description;
|
|
desc_file.close();
|
|
}
|
|
}
|
|
|
|
std::string subjects;
|
|
if (article.HasMember("artifact.ingress.subjects")) {
|
|
const rapidjson::Value &subjv =
|
|
article["artifact.ingress.subjects"];
|
|
if (subjv.IsArray()) {
|
|
for (auto i = 0; i < subjv.GetArray().Size(); i++) {
|
|
subjects += subjv[i].GetString();
|
|
subjects += " ";
|
|
}
|
|
std::replace(description.begin(), description.end(), ',', ':');
|
|
std::replace(description.begin(), description.end(), '\n', ' ');
|
|
}
|
|
}
|
|
|
|
std::string publishdate;
|
|
if (article.HasMember("artifact.publishedDate")) {
|
|
const rapidjson::Value &pubdatev =
|
|
article["artifact.publishedDate"];
|
|
if (pubdatev.IsString()) {
|
|
publishdate = pubdatev.GetString();
|
|
}
|
|
}
|
|
|
|
std::string license = "";
|
|
if (article.HasMember("artifact.ingress.license")) {
|
|
if (article["artifact.ingress.license"].IsString())
|
|
license = article["artifact.ingress.license"].GetString();
|
|
else if (article["artifact.ingress.license"].IsArray()) {
|
|
for (auto i = 0; i < article["artifact.ingress.license"].Size();
|
|
i++) {
|
|
license += article["artifact.ingress.license"][i].GetString();
|
|
}
|
|
}
|
|
}
|
|
|
|
bool is_cc_license = false;
|
|
if (license.find("CC by") != std::string::npos) {
|
|
is_cc_license = true;
|
|
char imgfetch[256];
|
|
snprintf(imgfetch, sizeof imgfetch, "wget %s -O \"%s-%s.jpeg\"",
|
|
imglink, article_id.c_str(), mediaid.c_str());
|
|
if (debug_level > 0)
|
|
std::cout << imgfetch << std::endl;
|
|
if (download_mode) {
|
|
std::cout << "running in download mode\n";
|
|
system(imgfetch);
|
|
if (debug_level > 0)
|
|
std::cout << "Found CC by license, fetched the image for "
|
|
<< mediaid << "\n";
|
|
}
|
|
}
|
|
|
|
if (!download_mode) {
|
|
if (is_cc_license) {
|
|
char exif_file[128];
|
|
snprintf(exif_file, sizeof exif_file, "%s-%s.jpeg.exif.json",
|
|
article_id.c_str(), mediaid.c_str());
|
|
// Open the file
|
|
FILE *exiffp = fopen(exif_file, "rb");
|
|
if (!exiffp) {
|
|
std::cerr << "Error: unable to open file"
|
|
<< std::string(exif_file) << std::endl;
|
|
return -1;
|
|
}
|
|
|
|
char exifreadBuffer[4096];
|
|
rapidjson::FileReadStream exifis(exiffp, exifreadBuffer,
|
|
sizeof(exifreadBuffer));
|
|
rapidjson::Document exifdoc;
|
|
exifdoc.ParseStream(exifis);
|
|
|
|
// Check if the document is valid
|
|
if (exifdoc.HasParseError()) {
|
|
std::cerr << "Error: failed to parse JSON document exif data"
|
|
<< std::endl;
|
|
}
|
|
fclose(exiffp);
|
|
|
|
std::string exif_model, exif_iso, exif_focallength,
|
|
exif_exposuretime, exif_aperture, exif_fnumber,
|
|
exif_datetimeoriginal;
|
|
if (exifdoc.IsArray()) {
|
|
if (exifdoc[0].IsObject()) {
|
|
if (exifdoc[0].GetObject().HasMember("Model")) {
|
|
exif_model = exifdoc[0].GetObject()["Model"].GetString();
|
|
}
|
|
if (exifdoc[0].GetObject().HasMember("ISO")) {
|
|
exif_iso =
|
|
std::to_string(exifdoc[0].GetObject()["ISO"].GetInt());
|
|
}
|
|
if (exifdoc[0].GetObject().HasMember("FocalLength")) {
|
|
exif_focallength =
|
|
exifdoc[0].GetObject()["FocalLength"].GetString();
|
|
}
|
|
if (exifdoc[0].GetObject().HasMember("ExposureTime")) {
|
|
if (exifdoc[0].GetObject()["ExposureTime"].IsString())
|
|
exif_exposuretime =
|
|
exifdoc[0].GetObject()["ExposureTime"].GetString();
|
|
}
|
|
if (exifdoc[0].GetObject().HasMember("ApertureValue")) {
|
|
exif_aperture = std::to_string(
|
|
exifdoc[0].GetObject()["ApertureValue"].GetDouble());
|
|
}
|
|
if (exifdoc[0].GetObject().HasMember("DateTimeOriginal")) {
|
|
exif_datetimeoriginal =
|
|
exifdoc[0].GetObject()["DateTimeOriginal"].GetString();
|
|
}
|
|
}
|
|
}
|
|
|
|
out_file1 <<
|
|
/* article["artifact.defaultMediaIdentifier"].GetString()<<
|
|
", "<< article["artifact.defaultPictureIndex"].GetInt()<<
|
|
", "<<
|
|
article["artifact.defaultPictureDimension"].GetString()<<
|
|
", "<<*/
|
|
article_id
|
|
<< ", " << title << ", " << yearstr(yearb) << ", "
|
|
<< yearstr(yeare) << ", " << description << ", "
|
|
<< itemlink << ", " << imglink << ", "
|
|
<< article_id + "-" + mediaid + ".jpeg"
|
|
<< ", " << subjects << ", " << publishdate << ", "
|
|
<< "Länge leve Kosta! exhibition"
|
|
<< ", "
|
|
<< "Kulturparken Småland / Smålands museum"
|
|
<< ", " << exif_model << ", " << exif_iso << ", "
|
|
<< exif_focallength << ", " << exif_exposuretime << ", "
|
|
<< exif_aperture << ", " << exif_datetimeoriginal
|
|
<< ", " << license << std::endl;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
out_file1.close();
|
|
return 0;
|
|
}
|