fix(script): Import dataframes directly from source

Renamed intermediate calculation vars from long and redundant names to
slightly shorter and more coherent versions.

.gitignore

@@ -1,6 +1,14 @@
-/03-documentation/DB
+/documentation/DB
-/03-documentation/library
+/documentation/library
-/04-outputs
+/output
+/manuscript/*_files/
+
+# Elsevier journal target:
+# Ignore bibliography style, spl (for natbib?) and latex class
+# since they don't get deleted after compilation
+*.spl
+*.bst
+*.cls
 
 # Created by https://www.toptal.com/developers/gitignore/api/-f,python,linux,vim,quarto,markdown,jupyternotebooks
 # Edit at https://www.toptal.com/developers/gitignore?templates=-f,python,linux,vim,quarto,markdown,jupyternotebooks

Makefile

@@ -8,6 +8,9 @@ render: scoping_review.qmd
 
 extract: scoping_review.qmd
 	poetry extract
+	poetry prisma
 
 release: scoping_review.qmd
+	poetry extract
+	poetry prisma
 	poetry milestone

README.md

@@ -1,18 +1,21 @@
 # Scoping Review: Inequalities on the Labour Market
 
-
 This repository contains all data, modelling and processing source code and the complete textual content to reproduce the scoping review study.
 The most up-to-date version of this repository can always be found [here](https://git.martyoeh.me/professional/wow-inequalities).
 
-Raw, intermediate and processed data can all be found in the `02-data/` directory:
+Raw references, extracted and processed data can all be found in the `data/` directory:
-Raw data include the unmodified database queries using the scoping review search terms.
+Reference data include the unmodified database queries using the scoping review search terms,
-Intermediate data are made up of the bibtex file produced by Zotero, after tagging and sorting in a Zotero library, ready to be re-imported into the application.
+and the bibtex file produced by Zotero after tagging and sorting in a Zotero library, ready to be re-imported into the application.
-Processed data include the fully extracted studies which make up the main sample for the review.
+Extracted data include the fully extracted studies which make up the main sample for the review.
+Processed data are ready to import into a dataframe or visualize in a report.
 
-The full article text and code can be found in the `scoping_review.qmd` file.
+All full texts and visualization code reside in the `manuscripts/` directory.
-It makes use of supplementary processing code which resides in the `src/` directory,
+The full working paper text and code can be found in the `scoping_review.qmd` file.
-mainly to load processed data from the `02-data/` directory and turn it into `.csv` data,
+The full article text can be found in the `article.qmd` file.
-as well as pre-processing those for visualization and validity ranking within the study.
+
+They both make use of supplementary extraction and processing code which resides in the `src/` directory,
+mainly to load processed data from the `data/` directory and turn it into `.csv` data,
+as well as pre-processing for visualization and validity ranking within the study.
 
 ## Execution and Reproduction
 
@@ -35,6 +38,6 @@ Now, by invoking `make` the project can be rendered:
 make
 ```
 
-Make will by default extract the processed data and use it to render the full project into a pdf, an html and a docx version of the review, which are deposited in the `04-outputs/` directory.
+Make will by default extract the processed data and use it to render the full project into a pdf, an html and a docx version of the review, which are deposited in the `outputs/` directory.
 
 You can invoke any of the `extract`, `render`, `release` steps manually instead by executing e.g. `make extract`.

_extensions/quarto-journals/elsevier/_extension.yml

@@ -0,0 +1,19 @@
+title: Elsevier Journal Format
+author: Charles Teague
+version: 0.4.2
+quarto-required: ">=1.2.198"
+contributes:
+  formats:
+    common:
+      date-format: full
+      filters:
+        - elsevier.lua
+    pdf:
+      documentclass: elsarticle
+      template-partials: ["partials/before-body.tex", "partials/title.tex"]
+      format-resources:
+        - elsarticle.cls
+      cite-method: natbib
+      number-sections: true
+    html:
+      theme: styles/elsevier.scss

_extensions/quarto-journals/elsevier/bib/elsevier-harvard.csl

@@ -0,0 +1,239 @@
+<?xml version="1.0" encoding="utf-8"?>
+<style xmlns="http://purl.org/net/xbiblio/csl" class="in-text" version="1.0" demote-non-dropping-particle="never" default-locale="en-US">
+  <info>
+    <title>Elsevier - Harvard (with titles)</title>
+    <id>http://www.zotero.org/styles/elsevier-harvard</id>
+    <link href="http://www.zotero.org/styles/elsevier-harvard" rel="self"/>
+    <link href="http://www.zotero.org/styles/ecology-letters" rel="template"/>
+    <link href="http://www.elsevier.com/journals/biological-conservation/0006-3207/guide-for-authors#68000" rel="documentation"/>
+    <author>
+      <name>David Kaplan</name>
+      <email>david.kaplan@ird.fr</email>
+    </author>
+    <contributor>
+      <name>Simon Kornblith</name>
+      <email>simon@simonster.com</email>
+    </contributor>
+    <contributor>
+      <name>Bruce D'Arcus</name>
+    </contributor>
+    <contributor>
+      <name>Curtis M. Humphrey</name>
+    </contributor>
+    <contributor>
+      <name>Richard Karnesky</name>
+      <email>karnesky+zotero@gmail.com</email>
+      <uri>http://arc.nucapt.northwestern.edu/Richard_Karnesky</uri>
+    </contributor>
+    <contributor>
+      <name>Sebastian Karcher</name>
+    </contributor>
+    <category citation-format="author-date"/>
+    <category field="biology"/>
+    <category field="generic-base"/>
+    <updated>2019-01-22T15:57:12+00:00</updated>
+    <rights license="http://creativecommons.org/licenses/by-sa/3.0/">This work is licensed under a Creative Commons Attribution-ShareAlike 3.0 License</rights>
+  </info>
+  <macro name="container">
+    <choose>
+      <if type="chapter paper-conference" match="any">
+        <text term="in" prefix=", " suffix=": "/>
+        <names variable="editor translator" delimiter=", " suffix=", ">
+          <name name-as-sort-order="all" sort-separator=", " initialize-with="." delimiter=", " delimiter-precedes-last="always"/>
+          <label form="short" text-case="capitalize-first" prefix=" (" suffix=")"/>
+        </names>
+        <group delimiter=", ">
+          <text variable="container-title" text-case="title"/>
+          <text variable="collection-title" text-case="title"/>
+        </group>
+      </if>
+      <else-if type="bill book graphic legal_case legislation motion_picture report song" match="any">
+        <group prefix=", " delimiter=", ">
+          <text variable="container-title"/>
+          <text variable="collection-title"/>
+        </group>
+      </else-if>
+      <else>
+        <group prefix=". " delimiter=", ">
+          <text variable="container-title" form="short"/>
+          <text variable="collection-title"/>
+        </group>
+      </else>
+    </choose>
+  </macro>
+  <macro name="author">
+    <names variable="author">
+      <name name-as-sort-order="all" sort-separator=", " initialize-with="." delimiter=", " delimiter-precedes-last="always"/>
+      <label form="short" prefix=" (" suffix=")" text-case="capitalize-first"/>
+      <substitute>
+        <names variable="editor"/>
+        <names variable="translator"/>
+        <text macro="title"/>
+      </substitute>
+    </names>
+  </macro>
+  <macro name="author-short">
+    <names variable="author">
+      <name form="short" and="text" delimiter=", " initialize-with=". "/>
+      <substitute>
+        <names variable="editor"/>
+        <names variable="translator"/>
+        <choose>
+          <if type="bill book graphic legal_case legislation motion_picture report song" match="any">
+            <text variable="title" form="short" font-style="italic"/>
+          </if>
+          <else>
+            <text variable="title" form="short" quotes="true"/>
+          </else>
+        </choose>
+      </substitute>
+    </names>
+  </macro>
+  <macro name="access">
+    <choose>
+      <if variable="DOI">
+        <text variable="DOI" prefix="https://doi.org/"/>
+      </if>
+      <else-if type="webpage post-weblog" match="any">
+        <group delimiter=" ">
+          <text value="URL"/>
+          <text variable="URL"/>
+          <group prefix="(" suffix=").">
+            <text term="accessed" suffix=" "/>
+            <date variable="accessed">
+              <date-part name="month" form="numeric" suffix="."/>
+              <date-part name="day" suffix="."/>
+              <date-part name="year" form="short"/>
+            </date>
+          </group>
+        </group>
+      </else-if>
+    </choose>
+  </macro>
+  <macro name="title">
+    <choose>
+      <if type="report thesis" match="any">
+        <text variable="title"/>
+        <group prefix=" (" suffix=")" delimiter=" ">
+          <text variable="genre"/>
+          <text variable="number" prefix="No. "/>
+        </group>
+      </if>
+      <else-if type="bill book graphic legal_case legislation motion_picture report song speech" match="any">
+        <text variable="title"/>
+        <text macro="edition" prefix=", "/>
+      </else-if>
+      <else-if type="webpage">
+        <text variable="title"/>
+        <text value="WWW Document" prefix=" [" suffix="]"/>
+      </else-if>
+      <else>
+        <text variable="title"/>
+      </else>
+    </choose>
+  </macro>
+  <macro name="publisher">
+    <group delimiter=", ">
+      <text variable="publisher"/>
+      <text variable="publisher-place"/>
+    </group>
+  </macro>
+  <macro name="event">
+    <choose>
+      <if variable="event">
+        <text term="presented at" text-case="capitalize-first" suffix=" "/>
+        <text variable="event"/>
+      </if>
+    </choose>
+  </macro>
+  <macro name="issued">
+    <choose>
+      <if variable="issued">
+        <date variable="issued">
+          <date-part name="year"/>
+        </date>
+      </if>
+      <else>
+        <text term="no date" form="short"/>
+      </else>
+    </choose>
+  </macro>
+  <macro name="edition">
+    <group delimiter=" ">
+      <choose>
+        <if is-numeric="edition">
+          <number variable="edition" form="ordinal"/>
+        </if>
+        <else>
+          <text variable="edition" suffix="."/>
+        </else>
+      </choose>
+      <text value="ed"/>
+    </group>
+  </macro>
+  <macro name="locators">
+    <choose>
+      <if type="article-journal article-magazine article-newspaper" match="any">
+        <group prefix=" " delimiter=", ">
+          <group>
+            <text variable="volume"/>
+          </group>
+          <text variable="page"/>
+        </group>
+      </if>
+      <else-if type="bill book graphic legal_case legislation motion_picture report song thesis" match="any">
+        <group delimiter=", " prefix=". ">
+          <text macro="event"/>
+          <text macro="publisher"/>
+        </group>
+      </else-if>
+      <else-if type="chapter paper-conference" match="any">
+        <group delimiter=", " prefix=". ">
+          <text macro="event"/>
+          <text macro="publisher"/>
+          <group>
+            <label variable="page" form="short" suffix=" "/>
+            <text variable="page"/>
+          </group>
+        </group>
+      </else-if>
+      <else-if type="patent">
+        <text variable="number" prefix=". "/>
+      </else-if>
+    </choose>
+  </macro>
+  <citation et-al-min="3" et-al-use-first="1" disambiguate-add-givenname="true" disambiguate-add-year-suffix="true" collapse="year" cite-group-delimiter=", ">
+    <sort>
+      <key macro="author"/>
+      <key macro="issued" sort="descending"/>
+    </sort>
+    <layout prefix="(" suffix=")" delimiter="; ">
+      <group delimiter=", ">
+        <text macro="author-short"/>
+        <text macro="issued"/>
+        <group delimiter=" ">
+          <label variable="locator" form="short"/>
+          <text variable="locator"/>
+        </group>
+      </group>
+    </layout>
+  </citation>
+  <bibliography hanging-indent="true" entry-spacing="0" line-spacing="1">
+    <sort>
+      <key macro="author"/>
+      <key macro="issued" sort="descending"/>
+    </sort>
+    <layout>
+      <group suffix=".">
+        <text macro="author" suffix=","/>
+        <text macro="issued" prefix=" "/>
+        <group prefix=". ">
+          <text macro="title"/>
+          <text macro="container"/>
+          <text macro="locators"/>
+        </group>
+      </group>
+      <text macro="access" prefix=". "/>
+    </layout>
+  </bibliography>
+</style>

_extensions/quarto-journals/elsevier/elsevier.lua

@@ -0,0 +1,171 @@
+-- cite style constants
+local kBibStyleDefault = 'number'
+local kBibStyles = { 'number', 'numbername', 'authoryear' }
+local kBibStyleAuthYr = 'elsarticle-harv'
+local kBibStyleNumber = 'elsarticle-num'
+local kBibStyleNumberName = 'elsarticle-num-names'
+local kBibStyleUnknown = kBibStyleNumberName
+
+-- layout and style
+local kFormatting = pandoc.List({ 'preprint', 'review', 'doubleblind' })
+local kModels = pandoc.List({ '1p', '3p', '5p' })
+local kLayouts = pandoc.List({ 'onecolumn', 'twocolumn' })
+
+
+local function setBibStyle(meta, style)
+  if meta['biblio-style'] == nil then
+    meta['biblio-style'] = style
+    quarto.doc.add_format_resource('bib/' .. style .. '.bst')
+  end
+end
+
+local function hasClassOption(meta, option)
+  if meta['classoption'] == nil then
+    return false
+  end
+
+  for i, v in ipairs(meta['classoption']) do
+    if v[1].text == option then
+      return true
+    end
+  end
+  return false
+end
+
+local function addClassOption(meta, option)
+  if meta['classoption'] == nil then
+    meta['classoption'] = pandoc.List({})
+  end
+
+  if not hasClassOption(meta, option) then
+    meta['classoption']:insert({ pandoc.Str(option) })
+  end
+end
+
+local function printList(list)
+  local result = ''
+  local sep = ''
+  for i, v in ipairs(list) do
+    result = result .. sep .. v
+    sep = ', '
+  end
+  return result
+end
+
+local bibstyle = kBibStyleDefault
+
+return {
+  {
+    Meta = function(meta)
+      -- If citeproc is being used, switch to the proper
+      -- CSL file
+      if quarto.doc.cite_method() == 'citeproc' and meta['csl'] == nil then
+        meta['csl'] = quarto.utils.resolve_path('bib/elsevier-harvard.csl')
+      end
+
+      if quarto.doc.is_format("pdf") then
+
+        -- read the journal settings
+        local journal = meta['journal']
+        local citestyle = nil
+        local formatting = nil
+        local model = nil
+        local layout = nil
+        local name = nil
+
+        if journal ~= nil then
+          citestyle = journal['cite-style']
+          formatting = journal['formatting']
+          model = journal['model']
+          layout = journal['layout']
+          name = journal['name']
+        end
+
+        -- process the site style
+        if citestyle ~= nil then
+          citestyle = pandoc.utils.stringify(citestyle)
+        else
+          citestyle = kBibStyleDefault
+        end
+
+        -- capture the bibstyle
+        bibstyle = citestyle
+        if citestyle == 'numbername' then
+          setBibStyle(meta, kBibStyleNumberName)
+          addClassOption(meta, 'number')
+        elseif citestyle == 'authoryear' then
+          setBibStyle(meta, kBibStyleAuthYr)
+          addClassOption(meta, 'authoryear')
+        elseif citestyle == 'number' then
+          setBibStyle(meta, kBibStyleNumber)
+          addClassOption(meta, 'number')
+        elseif citestyle == 'super' then
+          addClassOption(meta, 'super')          
+          setBibStyle(meta, kBibStyleNumber)
+        else
+          error("Unknown journal cite-style " .. citestyle .. "\nPlease use one of " .. printList(kBibStyles))
+          setBibStyle(meta, kBibStyleUnknown)
+        end
+
+        -- process the layout
+        if formatting ~= nil then
+          formatting = pandoc.utils.stringify(formatting)
+          if kFormatting:includes(formatting) then
+            addClassOption(meta, formatting)
+          else
+            error("Unknown journal formatting " .. formatting .. "\nPlease use one of " .. printList(kFormatting))
+          end
+        end
+
+        -- process the type
+        if model ~= nil then
+          model = pandoc.utils.stringify(model)
+          if kModels:includes(model) then
+            addClassOption(meta, model)
+          else
+            error("Unknown journal model " .. model .. "\nPlease use one of " .. printList(kModels))
+          end
+        end
+
+        -- 5p models should be two column always
+        if model == '5p' and layout == nil then
+          layout = 'twocolumn'
+        end
+
+        -- process the type
+        if layout ~= nil then
+          layout = pandoc.utils.stringify(layout)
+          if kLayouts:includes(layout) then
+            addClassOption(meta, layout)
+            if layout == 'twocolumn' then
+              quarto.doc.include_file('in-header', 'partials/_two-column-longtable.tex')
+            end
+          else
+            error("Unknown journal layout " .. layout .. "\nPlease use one of " .. printList(kLayouts))
+          end
+        end
+
+        -- process the name
+        if name ~= nil then
+          name = pandoc.utils.stringify(name)
+          quarto.doc.include_text('in-header', '\\journal{' .. name .. '}')
+        end
+      end
+
+      return meta
+    end
+  },
+  {
+    Cite = function(cite)
+      if bibstyle == 'number' then
+        -- If we are numbered, force citations into normal mode
+        -- as the author styles don't make sense
+        for i, v in ipairs(cite.citations) do
+          v.mode = 'NormalCitation'
+        end
+        return cite
+      end
+    end,
+
+  }
+}

_extensions/quarto-journals/elsevier/partials/_two-column-longtable.tex

@@ -0,0 +1,20 @@
+\usepackage{float}
+\makeatletter
+\let\oldlt\longtable
+\let\endoldlt\endlongtable
+\def\longtable{\@ifnextchar[\longtable@i \longtable@ii}
+\def\longtable@i[#1]{\begin{figure}[H]
+\onecolumn
+\begin{minipage}{0.5\textwidth}
+\oldlt[#1]
+}
+\def\longtable@ii{\begin{figure}[H]
+\onecolumn
+\begin{minipage}{0.5\textwidth}
+\oldlt
+}
+\def\endlongtable{\endoldlt
+\end{minipage}
+\twocolumn
+\end{figure}}
+\makeatother

_extensions/quarto-journals/elsevier/partials/before-body.tex

@@ -0,0 +1,41 @@
+
+\begin{frontmatter}
+\title{$title$$if(subtitle)$ \\\large{$subtitle$} $endif$}
+$for(by-author)$\author[$for(by-author.affiliations)$$it.number$$sep$,$endfor$]{$by-author.name.literal$%
+$if(by-author.attributes.corresponding)$\corref{cor1}$endif$%
+$if(by-author.note.text)$\fnref{fn$by-author.note.number$}$endif$}
+$if(by-author.email)$ \ead{$by-author.email$} $endif$
+$endfor$
+$for(by-affiliation)$
+\affiliation[$it.number$]{organization={$it.name$$if(it.department)$, $it.department$$endif$},$if(it.address)$addressline={$it.address$}$endif$$if(it.city)$$if(it.address)$,$endif$city={$it.city$}$endif$$if(it.country)$$if(it.city)$,$elseif(it.address)$,$endif$country={$it.country$},countrysep={,}$endif$$if(it.postal-code)$$if(it.country)$,$elseif(it.city)$,$elseif(it.address)$,$endif$postcode={$it.postal-code$}$endif$,postcodesep={}}
+$endfor$
+
+\cortext[cor1]{Corresponding author}
+$for(by-author)$
+$if(by-author.note.text)$\fntext[fn$by-author.note.number$]{$by-author.note.text$}$endif$
+$endfor$
+        
+$if(abstract)$
+\begin{abstract}
+$abstract$
+\end{abstract}
+$endif$
+
+$if(journal.graphical-abstract)$\begin{graphicalabstract}
+$journal.graphical-abstract$
+\end{graphicalabstract}
+$endif$
+
+$if(journal.highlights)$\begin{highlights}
+$for(journal.highlights)$\item $it$$endfor$
+\end{highlights}
+$endif$
+
+$if(keywords)$
+\begin{keyword}
+    $for(keywords/allbutlast)$$keywords$ \sep $endfor$
+    $for(keywords/last)$$keywords$$endfor$
+\end{keyword}
+$endif$
+\end{frontmatter}
+    

_extensions/quarto-journals/elsevier/partials/title.tex

@@ -0,0 +1 @@
+\setlength{\parindent}{6pt}

_extensions/quarto-journals/elsevier/styles/elsevier.scss

@@ -0,0 +1 @@
+/*--  scss:defaults --*/

_quarto-journal.yml

@@ -0,0 +1,40 @@
+project:
+  type: default
+  render:
+    - manuscript/article.qmd
+
+format:
+  elsevier-html:
+    toc: true
+    code-fold: true
+    code-tools: true
+    embed-resources: true
+    fig-format: svg
+    link-external-icon: true
+    number-sections: true
+    reference-links: true
+    theme: darkly
+  docx:
+    filters:
+      - pandoc-to-zotero-live
+      - docx-landscape
+    echo: false
+    number-sections: true
+    reference-doc: data/supplementary/justified.docx
+  elsevier-pdf:
+    echo: false
+    number-sections: true
+    # see https://github.com/quarto-journals/elsevier
+
+journal:
+  name: World Development
+  model: 3p
+  formatting: review
+  cite-style: authoryear
+  highlights:
+    - Inequality is bad
+    - Some policies help against inequality
+    - External validity and internal validity tend to have reciprocal relationship to each other
+
+# execute:
+#   freeze: true

_quarto-workingpaper.yml

@@ -0,0 +1,29 @@
+project:
+  render:
+    - manuscript/presentation_summary.md
+    - manuscript/notes.qmd
+    - manuscript/meeting_eoy.qmd
+    - manuscript/scoping_review.qmd
+
+toc: true
+format:
+  html:
+    code-fold: true
+    code-tools: true
+    embed-resources: true
+    fig-format: svg
+    link-external-icon: true
+    number-sections: false
+    reference-links: true
+    theme: darkly
+  docx:
+    echo: false
+    number-sections: true
+    reference-doc: data/supplementary/justified.docx
+    filters:
+      - pandoc-to-zotero-live
+      - docx-landscape
+  pdf:
+    echo: false
+    number-sections: true
+    pdf-engine: tectonic

_quarto.yml

@@ -1,35 +1,11 @@
 project:
-  title: "Key terms and definitions"
+  output-dir: output
-  output-dir: 04-outputs
+  execute-dir: project
-  render:
-    - presentation_summary.md
-    - notes.qmd
-    - meeting_eoy.qmd
-    - scoping_review.qmd
 
-toc: true
+profile:
-format:
+  default: journal
-  html:
-    code-fold: true
-    code-tools: true
-    embed-resources: true
-    fig-format: svg
-    link-external-icon: true
-    number-sections: false
-    reference-links: true
-    theme: darkly
-  docx:
-    echo: false
-    number-sections: true
-    reference-doc: 02-data/supplementary/justified.docx
-    filters:
-       - pandoc-to-zotero-live
-  pdf:
-    echo: false
-    number-sections: true
-    pdf-engine: tectonic
 
-bibliography: 02-data/intermediate/zotero-library.bib
+bibliography: data/references/zotero-library.bib
 csl: /home/marty/documents/library/utilities/styles/APA-7.csl
 zoterolive:
   library: wow-inequalities

data/processed/prisma.mmd

@@ -0,0 +1,15 @@
+
+    flowchart TD;
+        search_db["Records identified through database searching (n=1749)"] --> starting_sample;
+        search_prev["Records identified through other sources (n=2240)"] --> starting_sample["Starting sample (n=3989)"];
+
+        starting_sample -- "Duplicate removal (267 removed) "--> dedup["Records after duplicates removed (n=3723)"];
+
+        dedup -- "Title screening (1779 excluded)" --> title_screened["Records after titles screened (n=1944)"];
+
+        title_screened -- "Abstract screening (1506 excluded)"--> abstract_screened["Records after abstracts screened (n=438)"];
+
+        abstract_screened -- "  Language screening (2 excluded)  "--> language_screened["Records after language screened (n=436)"];
+
+        language_screened -- "  Full-text screening (383 excluded)  "--> full-text_screened["Full-text articles assessed for eligibility (n=52)"];
+