releaser: Add release notes to /docs for release of 0.57.1

[ci skip]

.circleci/config.yml

@@ -1,8 +1,9 @@
 defaults: &defaults
-  working_directory: /go/src/github.com/gohugoio
   docker:
-      - image: bepsays/ci-goreleaser:0.34.2-11
-    
+      - image: bepsays/ci-goreleaser:1.12-5
+  environment:
+    CGO_ENABLED: "0"
+
 version: 2
 jobs:
   build:
@@ -12,11 +13,11 @@ jobs:
           path: hugo
       - run:
             command: |
-                go get -d github.com/magefile/mage/...
                 git clone git@github.com:gohugoio/hugoDocs.git
                 cd hugo
-                mage vendor
-                mage check
+                go mod download
+                sleep 5
+                go test -p 1 ./...
       - persist_to_workspace:
           root: .
           paths: .
@@ -24,7 +25,7 @@ jobs:
     <<: *defaults
     steps:
       - attach_workspace:
-          at: /go/src/github.com/gohugoio
+          at: /root/project
       - run:
             command: |
                     cd hugo

.dockerignore

@@ -0,0 +1,9 @@
+*.md
+*.log
+*.txt
+.git
+.github
+.circleci
+docs
+examples
+Dockerfile

.gitattributes

@@ -0,0 +1,8 @@
+# Text files have auto line endings
+* text=auto
+
+# Go source files always have LF line endings
+*.go text eol=lf
+
+# SVG files should not be modified
+*.svg -text

.github/SUPPORT.md

@@ -0,0 +1,3 @@
+### Asking Support Questions
+
+We have an active [discussion forum](https://discourse.gohugo.io) where users and developers can ask questions. Please don't use the GitHub issue tracker to ask questions.

.gitignore

@@ -1,4 +1,4 @@
-hugo
+/hugo
 docs/public*
 /.idea
 hugo.exe
@@ -15,5 +15,11 @@ vendor/*/
 *.debug
 coverage*.out
 
+dock.sh
+
 GoBuilds
 dist
+
+resources/sunset.jpg
+
+vendor

.travis.yml

@@ -1,27 +1,46 @@
 language: go
 sudo: false
-dist: trusty
+dist: xenial
+env:
+  global:
+  - GOPROXY="https://proxy.golang.org"
+  - HUGO_BUILD_TAGS="extended"
 git:
   depth: false
 go:
-  - 1.9.5
-  - "1.10.1"
+  - "1.11.10"
+  - "1.12.5"
   - tip
 os:
   - linux
   - osx
+  - windows
 matrix:
   allow_failures:
     - go: tip
   fast_finish: true
+  exclude:
+  - os: windows
+    go: tip
+
 install:
+  - mkdir -p $HOME/src
+  - mv $TRAVIS_BUILD_DIR $HOME/src
+  - export TRAVIS_BUILD_DIR=$HOME/src/hugo
+  - cd $HOME/src/hugo
   - go get github.com/magefile/mage
-  - mage -v vendor
 script:
-  - mage -v hugoRace
+  - go mod download || true
+  - mage -v test
   - mage -v check
+  - mage -v hugo
   - ./hugo -s docs/
   - ./hugo --renderToMemory -s docs/
+  - df -h
+
 before_install:
+  - df -h
+    # https://travis-ci.community/t/go-cant-find-gcc-with-go1-11-1-on-windows/293/5
+  - if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then choco install mingw -y; export PATH=/c/tools/mingw64/bin:"$PATH"; fi
   - gem install asciidoctor
   - type asciidoctor

CONTRIBUTING.md

@@ -9,7 +9,7 @@ The Hugo community and maintainers are [very active](https://github.com/gohugoio
 
 *Note that this repository only contains the actual source code of Hugo. For **only** documentation-related pull requests / issues please refer to the [hugoDocs](https://github.com/gohugoio/hugoDocs) repository.*
 
-*Pull requests that contain changes on the code base **and** related documentation, e.g. for a new feature, shall remain a single, atomic one.*
+*Changes to the codebase **and** related documentation, e.g. for a new feature, should still use a single pull request.*
 
 ## Table of Contents
 
@@ -18,11 +18,8 @@ The Hugo community and maintainers are [very active](https://github.com/gohugoio
 * [Submitting Patches](#submitting-patches)
   * [Code Contribution Guidelines](#code-contribution-guidelines)
   * [Git Commit Message Guidelines](#git-commit-message-guidelines)
-  * [Vendored Dependencies](#vendored-dependencies)
   * [Fetching the Sources From GitHub](#fetching-the-sources-from-github)
-  * [Using Git Remotes](#using-git-remotes)
-  * [Build Hugo with Your Changes](#build-hugo-with-your-changes)
-  * [Updating the Hugo Sources](#updating-the-hugo-sources)
+  * [Building Hugo with Your Changes](#building-hugo-with-your-changes)
 
 ## Asking Support Questions
 
@@ -32,21 +29,24 @@ Please don't use the GitHub issue tracker to ask questions.
 ## Reporting Issues
 
 If you believe you have found a defect in Hugo or its documentation, use
-the GitHub [issue tracker](https://github.com/gohugoio/hugo/issues) to report the problem to the Hugo maintainers.
-If you're not sure if it's a bug or not, start by asking in the [discussion forum](https://discourse.gohugo.io).
-When reporting the issue, please provide the version of Hugo in use (`hugo version`) and your operating system.
+the GitHub [issue tracker](https://github.com/gohugoio/hugo/issues) to report
+the problem to the Hugo maintainers. If you're not sure if it's a bug or not,
+start by asking in the [discussion forum](https://discourse.gohugo.io).
+When reporting the issue, please provide the version of Hugo in use (`hugo
+version`) and your operating system.
 
 ## Code Contribution
 
-Hugo has become a fully featured static site generator, so any new functionality must meet these criterias:
+Hugo has become a fully featured static site generator, so any new functionality must:
 
-* It must be useful to many.
-* It must fit natural into _what Hugo does best._
-* It must strive not to break existing sites.
-* It must close ur update an open [Hugo issue](https://github.com/gohugoio/hugo/issues)
-* If it is of some complexity, the contributor is expected to maintain and support the new future (answer questions on the forum, fix any bugs etc.).
+* be useful to many.
+* fit naturally into _what Hugo does best._
+* strive not to break existing sites.
+* close or update an open [Hugo issue](https://github.com/gohugoio/hugo/issues)
 
-So, to avoid doing unneeded work, it is recommended to open up a discussion on the [Hugo Forum](https://discourse.gohugo.io/) to get some acceptance that this is a good idea. Also, if this is a complex feature, create a small design proposal on the [Hugo issue tracker](https://github.com/gohugoio/hugo/issues) before you start coding.
+If it is of some complexity, the contributor is expected to maintain and support the new future (answer questions on the forum, fix any bugs etc.).
+
+It is recommended to open up a discussion on the [Hugo Forum](https://discourse.gohugo.io/) to get feedback on your idea before you begin. If you are submitting a complex feature, create a small design proposal on the [Hugo issue tracker](https://github.com/gohugoio/hugo/issues) before you start.
 
 
 **Bug fixes are, of course, always welcome.**
@@ -55,16 +55,12 @@ So, to avoid doing unneeded work, it is recommended to open up a discussion on t
 
 ## Submitting Patches
 
-The Hugo project welcomes all contributors and contributions regardless of skill or experience level.
-If you are interested in helping with the project, we will help you with your contribution.
-Hugo is a very active project with many contributions happening daily.
-Because we want to create the best possible product for our users and the best contribution experience for our developers,
-we have a set of guidelines which ensure that all contributions are acceptable.
-The guidelines are not intended as a filter or barrier to participation.
-If you are unfamiliar with the contribution process, the Hugo team will help you and teach you how to bring your contribution in accordance with the guidelines.
+The Hugo project welcomes all contributors and contributions regardless of skill or experience level. If you are interested in helping with the project, we will help you with your contribution.
 
 ### Code Contribution Guidelines
 
+Because we want to create the best possible product for our users and the best contribution experience for our developers, we have a set of guidelines which ensure that all contributions are acceptable. The guidelines are not intended as a filter or barrier to participation. If you are unfamiliar with the contribution process, the Hugo team will help you and teach you how to bring your contribution in accordance with the guidelines.
+
 To make the contribution process as seamless as possible, we ask for the following:
 
 * Go ahead and fork the project and make your changes.  We encourage pull requests to allow for review and discussion of code changes.
@@ -74,7 +70,7 @@ To make the contribution process as seamless as possible, we ask for the followi
     * Run `go fmt`.
     * Add documentation if you are adding new features or changing functionality.  The docs site lives in `/docs`.
     * Squash your commits into a single commit. `git rebase -i`. It’s okay to force update your pull request with `git push -f`.
-    * Ensure that `mage check` succeeds. [Travis CI](https://travis-ci.org/gohugoio/hugo) (Linux and macOS) and [AppVeyor](https://ci.appveyor.com/project/gohugoio/hugo/branch/master) (Windows) will fail the build if `mage check` fails.
+    * Ensure that `mage check` succeeds. [Travis CI](https://travis-ci.org/gohugoio/hugo) (Windows, Linux and macOS) will fail the build if `mage check` fails.
     * Follow the **Git Commit Message Guidelines** below.
 
 ### Git Commit Message Guidelines
@@ -86,11 +82,11 @@ the most important part being that each commit message should have a title/subje
 Also, if your commit references one or more GitHub issues, always end your commit message body with *See #1234* or *Fixes #1234*.
 Replace *1234* with the GitHub issue ID. The last example will close the issue when the commit is merged into *master*.
 
-Sometimes it makes sense to prefix the commit message with the packagename (or docs folder) all lowercased ending with a colon.
+Sometimes it makes sense to prefix the commit message with the package name (or docs folder) all lowercased ending with a colon.
 That is fine, but the rest of the rules above apply.
 So it is "tpl: Add emojify template func", not "tpl: add emojify template func.", and "docs: Document emoji", not "doc: document emoji."
 
-Please consider to use a short and descriptive branch name, e.g. **NOT** "patch-1". It's very common but creates a naming conflict each time when a submission is pulled for a review.
+Please use a short and descriptive branch name, e.g. **NOT** "patch-1". It's very common but creates a naming conflict each time when a submission is pulled for a review.
 
 An example:
 
@@ -107,30 +103,25 @@ Fixes #1949
 
 ###  Fetching the Sources From GitHub
 
-Due to the way Go handles package imports, the best approach for working on a
-Hugo fork is to use Git Remotes.  Here's a simple walk-through for getting
-started:
+Since Hugo 0.48, Hugo uses the Go Modules support built into Go 1.11 to build. The easiest is to clone Hugo in a directory outside of `GOPATH`, as in the following example:
 
-1. Get the Hugo source:
+```bash
+mkdir $HOME/src
+cd $HOME/src
+git clone https://github.com/gohugoio/hugo.git
+cd hugo
+go install
+```
 
-    ```bash
-    go get -u -v -d github.com/gohugoio/hugo
-    ```
+>Note: Some Go tools may not be fully updated to support Go Modules yet. One example would be LiteIDE. Follow [this workaround](https://github.com/visualfc/liteide/issues/986#issuecomment-428117702) for how to continue to work with Hugo below `GOPATH`.
 
-1. Install Mage:
+For some convenient build and test targets, you also will want to install Mage:
 
-    ```bash
-    go get github.com/magefile/mage
-    ```
+```bash
+go get github.com/magefile/mage
+```
 
-1. Change to the Hugo source directory and fetch the dependencies:
-
-    ```bash
-    cd $HOME/go/src/github.com/gohugoio/hugo
-    mage vendor
-    ```
-
-    Note that Hugo uses [Go Dep](https://github.com/golang/dep) to vendor dependencies, rather than a a simple `go get`. We don't commit the vendored packages themselves to the Hugo git repository. The call to `mage vendor` takes care of all this for you.
+Now, to make a change to Hugo's source:
 
 1. Create a new branch for your changes (the branch name is arbitrary):
 
@@ -168,7 +159,7 @@ Hugo uses [mage](https://github.com/magefile/mage) to sync vendor dependencies,
 cd $HOME/go/src/github.com/gohugoio/hugo
 ```
 
-To build Hugo: 
+To build Hugo:
 
 ```bash
 mage hugo
@@ -193,14 +184,8 @@ To list all available commands along with descriptions:
 mage -l
 ```
 
-### Updating the Hugo Sources
-
-If you want to stay in sync with the Hugo repository, you can easily pull down
-the source changes, but you'll need to keep the vendored packages up-to-date as
-well.
+**Note:** From Hugo 0.43 we have added a build tag, `extended` that adds **SCSS support**. This needs a C compiler installed to build. You can enable this when building by:
 
 ```bash
-git pull
-mage vendor
-```
-
+HUGO_BUILD_TAGS=extended mage install
+````

Dockerfile

@@ -1,23 +1,33 @@
-FROM golang:1.9.0-alpine3.6 AS build
+# GitHub:       https://github.com/gohugoio
+# Twitter:      https://twitter.com/gohugoio
+# Website:      https://gohugo.io/
+
+FROM golang:1.11-stretch AS build
 
-RUN apk add --no-cache --virtual git musl-dev
-RUN go get github.com/golang/dep/cmd/dep
 
 WORKDIR /go/src/github.com/gohugoio/hugo
-ADD . /go/src/github.com/gohugoio/hugo/
-RUN dep ensure
-RUN go install -ldflags '-s -w'
+RUN apt-get install \
+    git gcc g++ binutils
+COPY . /go/src/github.com/gohugoio/hugo/
+ENV GO111MODULE=on
+RUN go get -d .
 
-FROM alpine:3.6
-RUN \
-  adduser -h /site -s /sbin/nologin -u 1000 -D hugo && \
-  apk add --no-cache \
-    dumb-init
-COPY --from=build /go/bin/hugo /bin/hugo
-USER    hugo
-WORKDIR /site
-VOLUME  /site
+ARG CGO=0
+ENV CGO_ENABLED=${CGO}
+ENV GOOS=linux
+
+# default non-existent build tag so -tags always has an arg
+ARG BUILD_TAGS="99notag"
+RUN go install -ldflags '-w -extldflags "-static"' -tags ${BUILD_TAGS}
+
+# ---
+
+FROM alpine:3.9
+RUN apk add --no-cache ca-certificates
+COPY --from=build /go/bin/hugo /hugo
+ARG  WORKDIR="/site"
+WORKDIR ${WORKDIR}
+VOLUME  ${WORKDIR}
 EXPOSE  1313
-
-ENTRYPOINT ["/usr/bin/dumb-init", "--", "hugo"]
+ENTRYPOINT [ "/hugo" ]
 CMD [ "--help" ]

Gopkg.lock

@@ -1,437 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  branch = "master"
-  name = "github.com/BurntSushi/toml"
-  packages = ["."]
-  revision = "a368813c5e648fee92e5f6c30e3944ff9d5e8895"
-
-[[projects]]
-  name = "github.com/PuerkitoBio/purell"
-  packages = ["."]
-  revision = "0bcb03f4b4d0a9428594752bd2a3b9aa0a9d4bd4"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/PuerkitoBio/urlesc"
-  packages = ["."]
-  revision = "de5bf2ad457846296e2031421a34e2568e304e35"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/alecthomas/chroma"
-  packages = [
-    ".",
-    "formatters",
-    "formatters/html",
-    "lexers",
-    "lexers/a",
-    "lexers/b",
-    "lexers/c",
-    "lexers/circular",
-    "lexers/d",
-    "lexers/e",
-    "lexers/f",
-    "lexers/g",
-    "lexers/h",
-    "lexers/i",
-    "lexers/internal",
-    "lexers/j",
-    "lexers/k",
-    "lexers/l",
-    "lexers/m",
-    "lexers/n",
-    "lexers/o",
-    "lexers/p",
-    "lexers/q",
-    "lexers/r",
-    "lexers/s",
-    "lexers/t",
-    "lexers/v",
-    "lexers/w",
-    "lexers/x",
-    "lexers/y",
-    "styles"
-  ]
-  revision = "6b1131c0069211a7729895c26f8c51765a4baf25"
-
-[[projects]]
-  name = "github.com/bep/debounce"
-  packages = ["."]
-  revision = "844797fa1dd9ba969d71b62797ff19d1e49d4eac"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/bep/gitmap"
-  packages = ["."]
-  revision = "012701e8669671499fc43e9792335a1dcbfe2afb"
-
-[[projects]]
-  name = "github.com/chaseadamsio/goorgeous"
-  packages = ["."]
-  revision = "dcf1ef873b8987bf12596fe6951c48347986eb2f"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/cpuguy83/go-md2man"
-  packages = ["md2man"]
-  revision = "20f5889cbdc3c73dbd2862796665e7c465ade7d1"
-  version = "v1.0.8"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/danwakefield/fnmatch"
-  packages = ["."]
-  revision = "cbb64ac3d964b81592e64f957ad53df015803288"
-
-[[projects]]
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/disintegration/imaging"
-  packages = ["."]
-  revision = "dd50a3ee9985ccd313a2f03c398fcaedc96dc707"
-  version = "v1.2.4"
-
-[[projects]]
-  name = "github.com/dlclark/regexp2"
-  packages = [
-    ".",
-    "syntax"
-  ]
-  revision = "487489b64fb796de2e55f4e8a4ad1e145f80e957"
-  version = "v1.1.6"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/eknkc/amber"
-  packages = [
-    ".",
-    "parser"
-  ]
-  revision = "cdade1c073850f4ffc70a829e31235ea6892853b"
-
-[[projects]]
-  name = "github.com/fortytw2/leaktest"
-  packages = ["."]
-  revision = "a5ef70473c97b71626b9abeda80ee92ba2a7de9e"
-  version = "v1.2.0"
-
-[[projects]]
-  name = "github.com/fsnotify/fsnotify"
-  packages = ["."]
-  revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
-  version = "v1.4.7"
-
-[[projects]]
-  name = "github.com/gobwas/glob"
-  packages = [
-    ".",
-    "compiler",
-    "match",
-    "syntax",
-    "syntax/ast",
-    "syntax/lexer",
-    "util/runes",
-    "util/strings"
-  ]
-  revision = "5ccd90ef52e1e632236f7326478d4faa74f99438"
-  version = "v0.2.3"
-
-[[projects]]
-  name = "github.com/gorilla/websocket"
-  packages = ["."]
-  revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/go-immutable-radix"
-  packages = ["."]
-  revision = "7f3cd4390caab3250a57f30efdb2a65dd7649ecf"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/golang-lru"
-  packages = ["simplelru"]
-  revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/hcl"
-  packages = [
-    ".",
-    "hcl/ast",
-    "hcl/parser",
-    "hcl/printer",
-    "hcl/scanner",
-    "hcl/strconv",
-    "hcl/token",
-    "json/parser",
-    "json/scanner",
-    "json/token"
-  ]
-  revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168"
-
-[[projects]]
-  name = "github.com/inconshreveable/mousetrap"
-  packages = ["."]
-  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
-  version = "v1.0"
-
-[[projects]]
-  name = "github.com/jdkato/prose"
-  packages = [
-    "internal/util",
-    "transform"
-  ]
-  revision = "20d3663d4bc9dd10d75abcde9d92e04b4861c674"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/kyokomi/emoji"
-  packages = ["."]
-  revision = "7e06b236c489543f53868841f188a294e3383eab"
-  version = "v1.5"
-
-[[projects]]
-  name = "github.com/magefile/mage"
-  packages = [
-    "mg",
-    "sh"
-  ]
-  revision = "2f974307b636f59c13b88704cf350a4772fef271"
-  version = "v1.0.2"
-
-[[projects]]
-  name = "github.com/magiconair/properties"
-  packages = ["."]
-  revision = "c3beff4c2358b44d0493c7dda585e7db7ff28ae6"
-  version = "v1.7.6"
-
-[[projects]]
-  name = "github.com/markbates/inflect"
-  packages = ["."]
-  revision = "a12c3aec81a6a938bf584a4bac567afed9256586"
-
-[[projects]]
-  name = "github.com/mattn/go-runewidth"
-  packages = ["."]
-  revision = "9e777a8366cce605130a531d2cd6363d07ad7317"
-  version = "v0.0.2"
-
-[[projects]]
-  name = "github.com/miekg/mmark"
-  packages = ["."]
-  revision = "fd2f6c1403b37925bd7fe13af05853b8ae58ee5f"
-  version = "v1.3.6"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/mitchellh/mapstructure"
-  packages = ["."]
-  revision = "00c29f56e2386353d58c599509e8dc3801b0d716"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/muesli/smartcrop"
-  packages = [
-    ".",
-    "options"
-  ]
-  revision = "f6ebaa786a12a0fdb2d7c6dee72808e68c296464"
-
-[[projects]]
-  name = "github.com/nicksnyder/go-i18n"
-  packages = [
-    "i18n/bundle",
-    "i18n/language",
-    "i18n/translation"
-  ]
-  revision = "0dc1626d56435e9d605a29875701721c54bc9bbd"
-  version = "v1.10.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/olekukonko/tablewriter"
-  packages = ["."]
-  revision = "b8a9be070da40449e501c3c4730a889e42d87a9e"
-
-[[projects]]
-  name = "github.com/pelletier/go-toml"
-  packages = ["."]
-  revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/russross/blackfriday"
-  packages = ["."]
-  revision = "55d61fa8aa702f59229e6cff85793c22e580eaf5"
-  version = "v1.5.1"
-
-[[projects]]
-  name = "github.com/sanity-io/litter"
-  packages = ["."]
-  revision = "ae543b7ba8fd6af63e4976198f146e1348ae53c1"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/shurcooL/sanitized_anchor_name"
-  packages = ["."]
-  revision = "86672fcb3f950f35f2e675df2240550f2a50762f"
-
-[[projects]]
-  name = "github.com/spf13/afero"
-  packages = [
-    ".",
-    "mem"
-  ]
-  revision = "63644898a8da0bc22138abf860edaf5277b6102e"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/spf13/cast"
-  packages = ["."]
-  revision = "8965335b8c7107321228e3e3702cab9832751bac"
-  version = "v1.2.0"
-
-[[projects]]
-  name = "github.com/spf13/cobra"
-  packages = [
-    ".",
-    "doc"
-  ]
-  revision = "a1f051bc3eba734da4772d60e2d677f47cf93ef4"
-  version = "v0.0.2"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/fsync"
-  packages = ["."]
-  revision = "12a01e648f05a938100a26858d2d59a120307a18"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/jwalterweatherman"
-  packages = ["."]
-  revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/nitro"
-  packages = ["."]
-  revision = "24d7ef30a12da0bdc5e2eb370a79c659ddccf0e8"
-
-[[projects]]
-  name = "github.com/spf13/pflag"
-  packages = ["."]
-  revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/spf13/viper"
-  packages = ["."]
-  revision = "b5e8006cbee93ec955a89ab31e0e3ce3204f3736"
-  version = "v1.0.2"
-
-[[projects]]
-  name = "github.com/stretchr/testify"
-  packages = [
-    "assert",
-    "require"
-  ]
-  revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
-  version = "v1.2.1"
-
-[[projects]]
-  name = "github.com/yosssi/ace"
-  packages = ["."]
-  revision = "ea038f4770b6746c3f8f84f14fa60d9fe1205b56"
-  version = "v0.0.5"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/image"
-  packages = [
-    "bmp",
-    "draw",
-    "math/f64",
-    "riff",
-    "tiff",
-    "tiff/lzw",
-    "vp8",
-    "vp8l",
-    "webp"
-  ]
-  revision = "f315e440302883054d0c2bd85486878cb4f8572c"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "idna"
-  ]
-  revision = "61147c48b25b599e5b561d2e9c4f3e1ef489ca41"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sync"
-  packages = ["errgroup"]
-  revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  revision = "3b87a42e500a6dc65dae1a55d0b641295971163e"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/text"
-  packages = [
-    "collate",
-    "collate/build",
-    "internal/colltab",
-    "internal/gen",
-    "internal/language",
-    "internal/language/compact",
-    "internal/tag",
-    "internal/triegen",
-    "internal/ucd",
-    "language",
-    "secure/bidirule",
-    "transform",
-    "unicode/bidi",
-    "unicode/cldr",
-    "unicode/norm",
-    "unicode/rangetable",
-    "width"
-  ]
-  revision = "2cb43934f0eece38629746959acc633cba083fe4"
-
-[[projects]]
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
-  version = "v2.2.1"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "b9b38ece73fd6c0d0b20eb2ff85dfcc15a82841223b4403566712fd7b7ae7d59"
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,151 +0,0 @@
-
-
-[[constraint]]
-  name = "github.com/BurntSushi/toml"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/PuerkitoBio/purell"
-  version = "1.1.0"
-
-[[constraint]]
-  name = "github.com/alecthomas/chroma"
-  branch = "master"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/bep/gitmap"
-  
-[[constraint]]
- name = "github.com/chaseadamsio/goorgeous"
- version = "^1.1.0"
-
-[[constraint]]
- name = "github.com/disintegration/imaging"
- version = "~v1.2.4"
-
-[[constraint]]
-  name = "github.com/magefile/mage"
-  version = "v1"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/eknkc/amber"
-
-[[constraint]]
-  name = "github.com/fortytw2/leaktest"
-  version = "1.1.0"
-
-[[constraint]]
-  name = "github.com/fsnotify/fsnotify"
-  version = "^1.4.0"
-
-[[constraint]]
-  name = "github.com/gorilla/websocket"
-  version = "1.2.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/hashicorp/go-immutable-radix"
-
-[[constraint]]
-  name = "github.com/jdkato/prose"
-  version = "1.1.0"
-
-[[constraint]]
-  name = "github.com/kyokomi/emoji"
-  version = "1.5.0"
-
-[[constraint]]
-  name = "github.com/markbates/inflect"
-  revision = "a12c3aec81a6a938bf584a4bac567afed9256586"
-
-[[constraint]]
-  name = "github.com/miekg/mmark"
-  version = "^1.3.6"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/mitchellh/mapstructure"
-
-[[constraint]]
-  name = "github.com/nicksnyder/go-i18n"
-  version = "^1.10.0"
-
-[[constraint]]
-  name = "github.com/russross/blackfriday"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/spf13/afero"
-  version = "^1.1.0"
-
-[[constraint]]
-  name = "github.com/spf13/cast"
-  version = "^1.1.0"
-
-[[constraint]]
-  version = "^0.0.1"
-  name = "github.com/spf13/cobra"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/spf13/fsync"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/spf13/jwalterweatherman"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/spf13/nitro"
-
-[[constraint]]
-  name = "github.com/spf13/pflag"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/spf13/viper"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/stretchr/testify"
-  version = "1.1.4"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/olekukonko/tablewriter"
-
-[[constraint]]
-  name = "github.com/yosssi/ace"
-  version = "^0.0.5"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/image"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/text"
-
-[[constraint]]
-  branch = "v2"
-  name = "gopkg.in/yaml.v2"
-
-[[constraint]]
-  name = "github.com/gobwas/glob"
-  version = "0.2.2"
-
-
-[[constraint]]
-  name = "github.com/muesli/smartcrop"
-  branch = "master"
-
-
-[[constraint]]
-  name = "github.com/sanity-io/litter"
-  version = "1.1.0"
-
-[[constraint]]
-  name = "github.com/bep/debounce"
-  version = "^1.1.0"

README.md

@@ -1,19 +1,16 @@
-![Hugo](https://raw.githubusercontent.com/gohugoio/hugoDocs/master/static/img/hugo-logo.png)
+<img src="https://raw.githubusercontent.com/gohugoio/gohugoioTheme/master/static/images/hugo-logo-wide.svg?sanitize=true" alt="Hugo" width="565">
 
 A Fast and Flexible Static Site Generator built with love by [bep](https://github.com/bep), [spf13](http://spf13.com/) and [friends](https://github.com/gohugoio/hugo/graphs/contributors) in [Go][].
 
 [Website](https://gohugo.io) |
 [Forum](https://discourse.gohugo.io) |
-[Developer Chat (no support)](https://gitter.im/gohugoio/hugo) |
-[Documentation](https://gohugo.io/overview/introduction/) |
-[Installation Guide](https://gohugo.io/overview/installing/) |
+[Documentation](https://gohugo.io/getting-started/) |
+[Installation Guide](https://gohugo.io/getting-started/installing/) |
 [Contribution Guide](CONTRIBUTING.md) |
-[Twitter](http://twitter.com/gohugoio)
+[Twitter](https://twitter.com/gohugoio)
 
 [![GoDoc](https://godoc.org/github.com/gohugoio/hugo?status.svg)](https://godoc.org/github.com/gohugoio/hugo)
-[![Linux and macOS Build Status](https://api.travis-ci.org/gohugoio/hugo.svg?branch=master&label=Linux+and+macOS+build "Linux and macOS Build Status")](https://travis-ci.org/gohugoio/hugo)
-[![Windows Build Status](https://ci.appveyor.com/api/projects/status/a5mr220vsd091kua?svg=true&label=Windows+build "Windows Build Status")](https://ci.appveyor.com/project/bep/hugo/branch/master)
-[![Dev chat at https://gitter.im/gohugoio/hugo](https://img.shields.io/badge/gitter-developer_chat-46bc99.svg)](https://gitter.im/spf13/hugo?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Linux and macOS Build Status](https://api.travis-ci.org/gohugoio/hugo.svg?branch=master&label=Windows+and+Linux+and+macOS+build "Windows, Linux and macOS Build Status")](https://travis-ci.org/gohugoio/hugo)
 [![Go Report Card](https://goreportcard.com/badge/github.com/gohugoio/hugo)](https://goreportcard.com/report/github.com/gohugoio/hugo)
 
 ## Overview
@@ -36,7 +33,7 @@ Currently, we provide pre-built Hugo binaries for Windows, Linux, FreeBSD, NetBS
 
 Hugo may also be compiled from source wherever the Go compiler tool chain can run, e.g. for other operating systems including DragonFly BSD, OpenBSD, Plan 9, and Solaris.
 
-**Complete documentation is available at [Hugo Documentation][].**
+**Complete documentation is available at [Hugo Documentation](https://gohugo.io/getting-started/).**
 
 ## Choose How to Install
 
@@ -50,29 +47,25 @@ Building the binaries is an easy task for an experienced `go` getter.
 
 ### Install Hugo as Your Site Generator (Binary Install)
 
-Use the [installation instructions in the Hugo documentation](https://gohugo.io/overview/installing/).
+Use the [installation instructions in the Hugo documentation](https://gohugo.io/getting-started/installing/).
 
 ### Build and Install the Binaries from Source (Advanced Install)
 
 #### Prerequisite Tools
 
-* [Git](http://git-scm.com/)
-* [Go (latest or previous version)](https://golang.org/dl/)
-
-#### Vendored Dependencies
-
-Hugo uses [dep](https://github.com/golang/dep) to vendor dependencies, but we don't commit the vendored packages themselves to the Hugo git repository. Therefore, a simple `go get` is _not_ supported because the command is not vendor aware.
-
-The simplest way is to use [mage](https://github.com/magefile/mage) (a Make alternative for Go projects.)
+* [Git](https://git-scm.com/)
+* [Go (at least Go 1.11)](https://golang.org/dl/)
 
 #### Fetch from GitHub
 
+Since Hugo 0.48, Hugo uses the Go Modules support built into Go 1.11 to build. The easiest is to clone Hugo in a directory outside of `GOPATH`, as in the following example:
+
 ```bash
-go get github.com/magefile/mage
-go get -d github.com/gohugoio/hugo
-cd ${GOPATH:-$HOME/go}/src/github.com/gohugoio/hugo
-mage vendor
-mage install
+mkdir $HOME/src
+cd $HOME/src
+git clone https://github.com/gohugoio/hugo.git
+cd hugo
+go install
 ```
 
 **If you are a Windows user, substitute the `$HOME` environment variable above with `%USERPROFILE%`.**
@@ -124,3 +117,71 @@ For a complete guide to contributing code to Hugo, see the [Contribution Guide](
 
 [Go]: https://golang.org/
 [Hugo Documentation]: https://gohugo.io/overview/introduction/
+
+## Dependencies
+
+Hugo stands on the shoulder of many great open source libraries, in lexical order:
+
+ | Dependency  | License |
+ | :------------- | :------------- |
+ | [github.com/BurntSushi/locker](https://github.com/BurntSushi/locker) | The Unlicense |
+ | [github.com/BurntSushi/toml](https://github.com/BurntSushi/toml) | MIT License |
+ | [github.com/PuerkitoBio/purell](https://github.com/PuerkitoBio/purell) | BSD 3-Clause "New" or "Revised" License |
+ | [github.com/PuerkitoBio/urlesc](https://github.com/PuerkitoBio/urlesc) | BSD 3-Clause "New" or "Revised" License |
+ | [github.com/alecthomas/chroma](https://github.com/alecthomas/chroma) |  MIT License |
+ | [github.com/bep/debounce](https://github.com/bep/debounce) |    MIT License |
+ | [github.com/bep/gitmap](https://github.com/bep/gitmap) |  MIT License |
+ | [github.com/bep/go-tocss](https://github.com/bep/go-tocss) | MIT License |
+ | [github.com/niklasfasching/go-org](https://github.com/niklasfasching/go-org) | MIT License |
+ | [github.com/cpuguy83/go-md2man](https://github.com/cpuguy83/go-md2man) | MIT License |
+ | [github.com/danwakefield/fnmatch](https://github.com/danwakefield/fnmatch) | BSD 2-Clause "Simplified" License |
+ | [github.com/disintegration/imaging](https://github.com/disintegration/imaging) |  MIT License |
+ | [github.com/dlclark/regexp2](https://github.com/dlclark/regexp2) | MIT License |
+ | [github.com/eknkc/amber](https://github.com/eknkc/amber) | MIT License |
+ | [github.com/fsnotify/fsnotify](https://github.com/fsnotify/fsnotify) |  BSD 3-Clause "New" or "Revised" License |
+ | [github.com/gobwas/glob](https://github.com/gobwas/glob) | MIT License |
+ | [github.com/gorilla/websocket](https://github.com/gorilla/websocket) |  BSD 2-Clause "Simplified" License |
+ | [github.com/hashicorp/go-immutable-radix](https://github.com/hashicorp/go-immutable-radix) | Mozilla Public License 2.0 |
+ | [github.com/hashicorp/golang-lru](https://github.com/hashicorp/golang-lru) | Mozilla Public License 2.0 |
+ | [github.com/hashicorp/hcl](https://github.com/hashicorp/hcl) |   Mozilla Public License 2.0 |
+ | [github.com/jdkato/prose](https://github.com/jdkato/prose) |    MIT License |
+ | [github.com/kyokomi/emoji](https://github.com/kyokomi/emoji) |   MIT License |
+ | [github.com/magiconair/properties](https://github.com/magiconair/properties) | BSD 2-Clause "Simplified" License |
+ | [github.com/markbates/inflect](https://github.com/markbates/inflect) |  MIT License |
+ | [github.com/mattn/go-isatty](https://github.com/mattn/go-isatty) | MIT License |
+ | [github.com/mattn/go-runewidth](https://github.com/mattn/go-runewidth) | MIT License |
+ | [github.com/miekg/mmark](https://github.com/miekg/mmark) | Simplified BSD License |
+ | [github.com/mitchellh/hashstructure](https://github.com/mitchellh/hashstructure) | MIT License |
+ | [github.com/mitchellh/mapstructure](https://github.com/mitchellh/mapstructure) | MIT License |
+ | [github.com/muesli/smartcrop](https://github.com/muesli/smartcrop) |   MIT License |
+ | [github.com/nicksnyder/go-i18n](https://github.com/nicksnyder/go-i18n) | MIT License |
+ | [github.com/olekukonko/tablewriter](https://github.com/olekukonko/tablewriter) | MIT License |
+ | [github.com/pelletier/go-toml](https://github.com/pelletier/go-toml) |  MIT License |
+ | [github.com/pkg/errors](https://github.com/pkg/errors) |  BSD 2-Clause "Simplified" License |
+ | [github.com/russross/blackfriday](https://github.com/russross/blackfriday)  | Simplified BSD License |
+ | [github.com/shurcooL/sanitized_anchor_name](https://github.com/shurcooL/sanitized_anchor_name) | MIT License |
+ | [github.com/spf13/afero](https://github.com/spf13/afero) | Apache License 2.0 |
+ | [github.com/spf13/cast](https://github.com/spf13/cast) |  MIT License |
+ | [github.com/spf13/cobra](https://github.com/spf13/cobra) | Apache License 2.0 |
+ | [github.com/spf13/fsync](https://github.com/spf13/fsync) | MIT License |
+ | [github.com/spf13/jwalterweatherman](https://github.com/spf13/jwalterweatherman) | MIT License |
+ | [github.com/spf13/nitro](https://github.com/spf13/nitro) | Apache License 2.0 |
+ | [github.com/spf13/pflag](https://github.com/spf13/pflag) | BSD 3-Clause "New" or "Revised" License |
+ | [github.com/spf13/viper](https://github.com/spf13/viper) | MIT License |
+ | [github.com/tdewolff/minify](https://github.com/tdewolff/minify) | MIT License |
+ | [github.com/tdewolff/parse](https://github.com/tdewolff/parse) | MIT License |
+ | [github.com/wellington/go-libsass](https://github.com/wellington/go-libsass) | Apache License 2.0 |
+ | [github.com/yosssi/ace](https://github.com/yosssi/ace) |  MIT License |
+ | [golang.org/x/image](https://golang.org/x/image) | BSD 3-Clause "New" or "Revised" License |
+ | [golang.org/x/net](https://golang.org/x/net) | BSD 3-Clause "New" or "Revised" License |
+ | [golang.org/x/sync](https://golang.org/x/sync) | BSD 3-Clause "New" or "Revised" License |
+ | [golang.org/x/sys](https://golang.org/x/sys) | BSD 3-Clause "New" or "Revised" License |
+ | [golang.org/x/text](https://golang.org/x/text) | BSD 3-Clause "New" or "Revised" License
+ | [gopkg.in/yaml.v2](https://gopkg.in/yaml.v2) |    Apache License 2.0 |
+
+  
+ 
+  
+ 
+ 
+ 

appveyor.yml

@@ -1,18 +0,0 @@
-init:
-  - set PATH=%PATH%;C:\MinGW\bin;%GOPATH%\bin
-  - go version
-  - go env
-
-# clones and cd's to path
-clone_folder: C:\GOPATH\src\github.com\gohugoio\hugo
-
-install:
-  - gem install asciidoctor
-  - pip install docutils
-  - go get github.com/magefile/mage
-
-build_script:
-  - mage vendor hugoRace
-  - mage -v check
-  - hugo -s docs/
-  - hugo --renderToMemory -s docs/

benchbep.sh

@@ -0,0 +1 @@
+gobench -package=./hugolib -bench="BenchmarkSiteNew/Deep_content_tree"

bepdock.sh

@@ -0,0 +1 @@
+docker run --rm --mount type=bind,source="$(pwd)",target=/hugo -w /hugo  -i -t bepsays/ci-goreleaser:1.11-2 /bin/bash

bufferpool/bufpool_test.go

@@ -14,14 +14,18 @@
 package bufferpool
 
 import (
-	"github.com/stretchr/testify/assert"
 	"testing"
+
+	qt "github.com/frankban/quicktest"
 )
 
 func TestBufferPool(t *testing.T) {
+	c := qt.New(t)
+
 	buff := GetBuffer()
 	buff.WriteString("do be do be do")
-	assert.Equal(t, "do be do be do", buff.String())
+	c.Assert(buff.String(), qt.Equals, "do be do be do")
 	PutBuffer(buff)
-	assert.Equal(t, 0, buff.Len())
+
+	c.Assert(buff.Len(), qt.Equals, 0)
 }

cache/filecache/filecache.go

@@ -0,0 +1,366 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gohugoio/hugo/common/hugio"
+
+	"github.com/gohugoio/hugo/helpers"
+
+	"github.com/BurntSushi/locker"
+	"github.com/spf13/afero"
+)
+
+const (
+	filecacheRootDirname = "filecache"
+)
+
+// Cache caches a set of files in a directory. This is usually a file on
+// disk, but since this is backed by an Afero file system, it can be anything.
+type Cache struct {
+	Fs afero.Fs
+
+	// Max age for items in this cache. Negative duration means forever,
+	// 0 is effectively turning this cache off.
+	maxAge time.Duration
+
+	// When set, we just remove this entire root directory on expiration.
+	pruneAllRootDir string
+
+	nlocker *lockTracker
+}
+
+type lockTracker struct {
+	seenMu sync.RWMutex
+	seen   map[string]struct{}
+
+	*locker.Locker
+}
+
+// Lock tracks the ids in use. We use this information to do garbage collection
+// after a Hugo build.
+func (l *lockTracker) Lock(id string) {
+	l.seenMu.RLock()
+	if _, seen := l.seen[id]; !seen {
+		l.seenMu.RUnlock()
+		l.seenMu.Lock()
+		l.seen[id] = struct{}{}
+		l.seenMu.Unlock()
+	} else {
+		l.seenMu.RUnlock()
+	}
+
+	l.Locker.Lock(id)
+}
+
+// ItemInfo contains info about a cached file.
+type ItemInfo struct {
+	// This is the file's name relative to the cache's filesystem.
+	Name string
+}
+
+// NewCache creates a new file cache with the given filesystem and max age.
+func NewCache(fs afero.Fs, maxAge time.Duration, pruneAllRootDir string) *Cache {
+	return &Cache{
+		Fs:              fs,
+		nlocker:         &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})},
+		maxAge:          maxAge,
+		pruneAllRootDir: pruneAllRootDir,
+	}
+}
+
+// lockedFile is a file with a lock that is released on Close.
+type lockedFile struct {
+	afero.File
+	unlock func()
+}
+
+func (l *lockedFile) Close() error {
+	defer l.unlock()
+	return l.File.Close()
+}
+
+// WriteCloser returns a transactional writer into the cache.
+// It's important that it's closed when done.
+func (c *Cache) WriteCloser(id string) (ItemInfo, io.WriteCloser, error) {
+	id = cleanID(id)
+	c.nlocker.Lock(id)
+
+	info := ItemInfo{Name: id}
+
+	f, err := helpers.OpenFileForWriting(c.Fs, id)
+	if err != nil {
+		c.nlocker.Unlock(id)
+		return info, nil, err
+	}
+
+	return info, &lockedFile{
+		File:   f,
+		unlock: func() { c.nlocker.Unlock(id) },
+	}, nil
+}
+
+// ReadOrCreate tries to lookup the file in cache.
+// If found, it is passed to read and then closed.
+// If not found a new file is created and passed to create, which should close
+// it when done.
+func (c *Cache) ReadOrCreate(id string,
+	read func(info ItemInfo, r io.Reader) error,
+	create func(info ItemInfo, w io.WriteCloser) error) (info ItemInfo, err error) {
+	id = cleanID(id)
+
+	c.nlocker.Lock(id)
+	defer c.nlocker.Unlock(id)
+
+	info = ItemInfo{Name: id}
+
+	if r := c.getOrRemove(id); r != nil {
+		err = read(info, r)
+		defer r.Close()
+		return
+	}
+
+	f, err := helpers.OpenFileForWriting(c.Fs, id)
+	if err != nil {
+		return
+	}
+
+	err = create(info, f)
+
+	return
+
+}
+
+// GetOrCreate tries to get the file with the given id from cache. If not found or expired, create will
+// be invoked and the result cached.
+// This method is protected by a named lock using the given id as identifier.
+func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (ItemInfo, io.ReadCloser, error) {
+	id = cleanID(id)
+
+	c.nlocker.Lock(id)
+	defer c.nlocker.Unlock(id)
+
+	info := ItemInfo{Name: id}
+
+	if r := c.getOrRemove(id); r != nil {
+		return info, r, nil
+	}
+
+	r, err := create()
+	if err != nil {
+		return info, nil, err
+	}
+
+	if c.maxAge == 0 {
+		// No caching.
+		return info, hugio.ToReadCloser(r), nil
+	}
+
+	var buff bytes.Buffer
+	return info,
+		hugio.ToReadCloser(&buff),
+		afero.WriteReader(c.Fs, id, io.TeeReader(r, &buff))
+}
+
+// GetOrCreateBytes is the same as GetOrCreate, but produces a byte slice.
+func (c *Cache) GetOrCreateBytes(id string, create func() ([]byte, error)) (ItemInfo, []byte, error) {
+	id = cleanID(id)
+
+	c.nlocker.Lock(id)
+	defer c.nlocker.Unlock(id)
+
+	info := ItemInfo{Name: id}
+
+	if r := c.getOrRemove(id); r != nil {
+		defer r.Close()
+		b, err := ioutil.ReadAll(r)
+		return info, b, err
+	}
+
+	b, err := create()
+	if err != nil {
+		return info, nil, err
+	}
+
+	if c.maxAge == 0 {
+		return info, b, nil
+	}
+
+	if err := afero.WriteReader(c.Fs, id, bytes.NewReader(b)); err != nil {
+		return info, nil, err
+	}
+	return info, b, nil
+
+}
+
+// GetBytes gets the file content with the given id from the cahce, nil if none found.
+func (c *Cache) GetBytes(id string) (ItemInfo, []byte, error) {
+	id = cleanID(id)
+
+	c.nlocker.Lock(id)
+	defer c.nlocker.Unlock(id)
+
+	info := ItemInfo{Name: id}
+
+	if r := c.getOrRemove(id); r != nil {
+		defer r.Close()
+		b, err := ioutil.ReadAll(r)
+		return info, b, err
+	}
+
+	return info, nil, nil
+}
+
+// Get gets the file with the given id from the cahce, nil if none found.
+func (c *Cache) Get(id string) (ItemInfo, io.ReadCloser, error) {
+	id = cleanID(id)
+
+	c.nlocker.Lock(id)
+	defer c.nlocker.Unlock(id)
+
+	info := ItemInfo{Name: id}
+
+	r := c.getOrRemove(id)
+
+	return info, r, nil
+}
+
+// getOrRemove gets the file with the given id. If it's expired, it will
+// be removed.
+func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser {
+	if c.maxAge == 0 {
+		// No caching.
+		return nil
+	}
+
+	if c.maxAge > 0 {
+		fi, err := c.Fs.Stat(id)
+		if err != nil {
+			return nil
+		}
+
+		if c.isExpired(fi.ModTime()) {
+			c.Fs.Remove(id)
+			return nil
+		}
+	}
+
+	f, err := c.Fs.Open(id)
+
+	if err != nil {
+		return nil
+	}
+
+	return f
+}
+
+func (c *Cache) isExpired(modTime time.Time) bool {
+	if c.maxAge < 0 {
+		return false
+	}
+	return c.maxAge == 0 || time.Since(modTime) > c.maxAge
+}
+
+// For testing
+func (c *Cache) getString(id string) string {
+	id = cleanID(id)
+
+	c.nlocker.Lock(id)
+	defer c.nlocker.Unlock(id)
+
+	f, err := c.Fs.Open(id)
+
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+
+	b, _ := ioutil.ReadAll(f)
+	return string(b)
+
+}
+
+// Caches is a named set of caches.
+type Caches map[string]*Cache
+
+// Get gets a named cache, nil if none found.
+func (f Caches) Get(name string) *Cache {
+	return f[strings.ToLower(name)]
+}
+
+// NewCaches creates a new set of file caches from the given
+// configuration.
+func NewCaches(p *helpers.PathSpec) (Caches, error) {
+	var dcfg Configs
+	if c, ok := p.Cfg.Get("filecacheConfigs").(Configs); ok {
+		dcfg = c
+	} else {
+		var err error
+		dcfg, err = DecodeConfig(p.Fs.Source, p.Cfg)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	fs := p.Fs.Source
+
+	m := make(Caches)
+	for k, v := range dcfg {
+		var cfs afero.Fs
+
+		if v.isResourceDir {
+			cfs = p.BaseFs.ResourcesCache
+		} else {
+			cfs = fs
+		}
+
+		if cfs == nil {
+			// TODO(bep) we still have some places that do not initialize the
+			// full dependencies of a site, e.g. the import Jekyll command.
+			// That command does not need these caches, so let us just continue
+			// for now.
+			continue
+		}
+
+		baseDir := v.Dir
+
+		if err := cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) {
+			return nil, err
+		}
+
+		bfs := afero.NewBasePathFs(cfs, baseDir)
+
+		var pruneAllRootDir string
+		if k == cacheKeyModules {
+			pruneAllRootDir = "pkg"
+		}
+
+		m[k] = NewCache(bfs, v.MaxAge, pruneAllRootDir)
+	}
+
+	return m, nil
+}
+
+func cleanID(name string) string {
+	return strings.TrimPrefix(filepath.Clean(name), helpers.FilePathSeparator)
+}

cache/filecache/filecache_config.go

@@ -0,0 +1,230 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/gohugoio/hugo/config"
+
+	"github.com/gohugoio/hugo/helpers"
+
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/spf13/afero"
+)
+
+const (
+	cachesConfigKey = "caches"
+
+	resourcesGenDir = ":resourceDir/_gen"
+)
+
+var defaultCacheConfig = Config{
+	MaxAge: -1, // Never expire
+	Dir:    ":cacheDir/:project",
+}
+
+const (
+	cacheKeyGetJSON = "getjson"
+	cacheKeyGetCSV  = "getcsv"
+	cacheKeyImages  = "images"
+	cacheKeyAssets  = "assets"
+	cacheKeyModules = "modules"
+)
+
+type Configs map[string]Config
+
+func (c Configs) CacheDirModules() string {
+	return c[cacheKeyModules].Dir
+}
+
+var defaultCacheConfigs = Configs{
+	cacheKeyModules: {
+		MaxAge: -1,
+		Dir:    ":cacheDir/modules",
+	},
+	cacheKeyGetJSON: defaultCacheConfig,
+	cacheKeyGetCSV:  defaultCacheConfig,
+	cacheKeyImages: {
+		MaxAge: -1,
+		Dir:    resourcesGenDir,
+	},
+	cacheKeyAssets: {
+		MaxAge: -1,
+		Dir:    resourcesGenDir,
+	},
+}
+
+type Config struct {
+	// Max age of cache entries in this cache. Any items older than this will
+	// be removed and not returned from the cache.
+	// a negative value means forever, 0 means cache is disabled.
+	MaxAge time.Duration
+
+	// The directory where files are stored.
+	Dir string
+
+	// Will resources/_gen will get its own composite filesystem that
+	// also checks any theme.
+	isResourceDir bool
+}
+
+// GetJSONCache gets the file cache for getJSON.
+func (f Caches) GetJSONCache() *Cache {
+	return f[cacheKeyGetJSON]
+}
+
+// GetCSVCache gets the file cache for getCSV.
+func (f Caches) GetCSVCache() *Cache {
+	return f[cacheKeyGetCSV]
+}
+
+// ImageCache gets the file cache for processed images.
+func (f Caches) ImageCache() *Cache {
+	return f[cacheKeyImages]
+}
+
+// ModulesCache gets the file cache for Hugo Modules.
+func (f Caches) ModulesCache() *Cache {
+	return f[cacheKeyModules]
+}
+
+// AssetsCache gets the file cache for assets (processed resources, SCSS etc.).
+func (f Caches) AssetsCache() *Cache {
+	return f[cacheKeyAssets]
+}
+
+func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) {
+	c := make(Configs)
+	valid := make(map[string]bool)
+	// Add defaults
+	for k, v := range defaultCacheConfigs {
+		c[k] = v
+		valid[k] = true
+	}
+
+	m := cfg.GetStringMap(cachesConfigKey)
+
+	_, isOsFs := fs.(*afero.OsFs)
+
+	for k, v := range m {
+		cc := defaultCacheConfig
+
+		dc := &mapstructure.DecoderConfig{
+			Result:           &cc,
+			DecodeHook:       mapstructure.StringToTimeDurationHookFunc(),
+			WeaklyTypedInput: true,
+		}
+
+		decoder, err := mapstructure.NewDecoder(dc)
+		if err != nil {
+			return c, err
+		}
+
+		if err := decoder.Decode(v); err != nil {
+			return nil, err
+		}
+
+		if cc.Dir == "" {
+			return c, errors.New("must provide cache Dir")
+		}
+
+		name := strings.ToLower(k)
+		if !valid[name] {
+			return nil, errors.Errorf("%q is not a valid cache name", name)
+		}
+
+		c[name] = cc
+	}
+
+	// This is a very old flag in Hugo, but we need to respect it.
+	disabled := cfg.GetBool("ignoreCache")
+
+	for k, v := range c {
+		dir := filepath.ToSlash(filepath.Clean(v.Dir))
+		hadSlash := strings.HasPrefix(dir, "/")
+		parts := strings.Split(dir, "/")
+
+		for i, part := range parts {
+			if strings.HasPrefix(part, ":") {
+				resolved, isResource, err := resolveDirPlaceholder(fs, cfg, part)
+				if err != nil {
+					return c, err
+				}
+				if isResource {
+					v.isResourceDir = true
+				}
+				parts[i] = resolved
+			}
+		}
+
+		dir = path.Join(parts...)
+		if hadSlash {
+			dir = "/" + dir
+		}
+		v.Dir = filepath.Clean(filepath.FromSlash(dir))
+
+		if !v.isResourceDir {
+			if isOsFs && !filepath.IsAbs(v.Dir) {
+				return c, errors.Errorf("%q must resolve to an absolute directory", v.Dir)
+			}
+
+			// Avoid cache in root, e.g. / (Unix) or c:\ (Windows)
+			if len(strings.TrimPrefix(v.Dir, filepath.VolumeName(v.Dir))) == 1 {
+				return c, errors.Errorf("%q is a root folder and not allowed as cache dir", v.Dir)
+			}
+		}
+
+		if !strings.HasPrefix(v.Dir, "_gen") {
+			// We do cache eviction (file removes) and since the user can set
+			// his/hers own cache directory, we really want to make sure
+			// we do not delete any files that do not belong to this cache.
+			// We do add the cache name as the root, but this is an extra safe
+			// guard. We skip the files inside /resources/_gen/ because
+			// that would be breaking.
+			v.Dir = filepath.Join(v.Dir, filecacheRootDirname, k)
+		} else {
+			v.Dir = filepath.Join(v.Dir, k)
+		}
+
+		if disabled {
+			v.MaxAge = 0
+		}
+
+		c[k] = v
+	}
+
+	return c, nil
+}
+
+// Resolves :resourceDir => /myproject/resources etc., :cacheDir => ...
+func resolveDirPlaceholder(fs afero.Fs, cfg config.Provider, placeholder string) (cacheDir string, isResource bool, err error) {
+	workingDir := cfg.GetString("workingDir")
+
+	switch strings.ToLower(placeholder) {
+	case ":resourcedir":
+		return "", true, nil
+	case ":cachedir":
+		d, err := helpers.GetCacheDir(fs, cfg)
+		return d, false, err
+	case ":project":
+		return filepath.Base(workingDir), false, nil
+	}
+
+	return "", false, errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder)
+}

cache/filecache/filecache_config_test.go

@@ -0,0 +1,196 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/spf13/afero"
+
+	"github.com/gohugoio/hugo/config"
+
+	qt "github.com/frankban/quicktest"
+	"github.com/spf13/viper"
+)
+
+func TestDecodeConfig(t *testing.T) {
+	t.Parallel()
+
+	c := qt.New(t)
+
+	configStr := `
+resourceDir = "myresources"
+contentDir = "content"
+dataDir = "data"
+i18nDir = "i18n"
+layoutDir = "layouts"
+assetDir = "assets"
+archetypeDir = "archetypes"
+
+[caches]
+[caches.getJSON]
+maxAge = "10m"
+dir = "/path/to/c1"
+[caches.getCSV]
+maxAge = "11h"
+dir = "/path/to/c2"
+[caches.images]
+dir = "/path/to/c3"
+
+`
+
+	cfg, err := config.FromConfigString(configStr, "toml")
+	c.Assert(err, qt.IsNil)
+	fs := afero.NewMemMapFs()
+	decoded, err := DecodeConfig(fs, cfg)
+	c.Assert(err, qt.IsNil)
+
+	c.Assert(len(decoded), qt.Equals, 5)
+
+	c2 := decoded["getcsv"]
+	c.Assert(c2.MaxAge.String(), qt.Equals, "11h0m0s")
+	c.Assert(c2.Dir, qt.Equals, filepath.FromSlash("/path/to/c2/filecache/getcsv"))
+
+	c3 := decoded["images"]
+	c.Assert(c3.MaxAge, qt.Equals, time.Duration(-1))
+	c.Assert(c3.Dir, qt.Equals, filepath.FromSlash("/path/to/c3/filecache/images"))
+
+}
+
+func TestDecodeConfigIgnoreCache(t *testing.T) {
+	t.Parallel()
+
+	c := qt.New(t)
+
+	configStr := `
+resourceDir = "myresources"
+contentDir = "content"
+dataDir = "data"
+i18nDir = "i18n"
+layoutDir = "layouts"
+assetDir = "assets"
+archeTypedir = "archetypes"
+
+ignoreCache = true
+[caches]
+[caches.getJSON]
+maxAge = 1234
+dir = "/path/to/c1"
+[caches.getCSV]
+maxAge = 3456
+dir = "/path/to/c2"
+[caches.images]
+dir = "/path/to/c3"
+
+`
+
+	cfg, err := config.FromConfigString(configStr, "toml")
+	c.Assert(err, qt.IsNil)
+	fs := afero.NewMemMapFs()
+	decoded, err := DecodeConfig(fs, cfg)
+	c.Assert(err, qt.IsNil)
+
+	c.Assert(len(decoded), qt.Equals, 5)
+
+	for _, v := range decoded {
+		c.Assert(v.MaxAge, qt.Equals, time.Duration(0))
+	}
+
+}
+
+func TestDecodeConfigDefault(t *testing.T) {
+	c := qt.New(t)
+	cfg := newTestConfig()
+
+	if runtime.GOOS == "windows" {
+		cfg.Set("resourceDir", "c:\\cache\\resources")
+		cfg.Set("cacheDir", "c:\\cache\\thecache")
+
+	} else {
+		cfg.Set("resourceDir", "/cache/resources")
+		cfg.Set("cacheDir", "/cache/thecache")
+	}
+
+	fs := afero.NewMemMapFs()
+
+	decoded, err := DecodeConfig(fs, cfg)
+
+	c.Assert(err, qt.IsNil)
+
+	c.Assert(len(decoded), qt.Equals, 5)
+
+	imgConfig := decoded[cacheKeyImages]
+	jsonConfig := decoded[cacheKeyGetJSON]
+
+	if runtime.GOOS == "windows" {
+		c.Assert(imgConfig.Dir, qt.Equals, filepath.FromSlash("_gen/images"))
+	} else {
+		c.Assert(imgConfig.Dir, qt.Equals, "_gen/images")
+		c.Assert(jsonConfig.Dir, qt.Equals, "/cache/thecache/hugoproject/filecache/getjson")
+	}
+
+	c.Assert(imgConfig.isResourceDir, qt.Equals, true)
+	c.Assert(jsonConfig.isResourceDir, qt.Equals, false)
+}
+
+func TestDecodeConfigInvalidDir(t *testing.T) {
+	t.Parallel()
+
+	c := qt.New(t)
+
+	configStr := `
+resourceDir = "myresources"
+contentDir = "content"
+dataDir = "data"
+i18nDir = "i18n"
+layoutDir = "layouts"
+assetDir = "assets"
+archeTypedir = "archetypes"
+
+[caches]
+[caches.getJSON]
+maxAge = "10m"
+dir = "/"
+
+`
+	if runtime.GOOS == "windows" {
+		configStr = strings.Replace(configStr, "/", "c:\\\\", 1)
+	}
+
+	cfg, err := config.FromConfigString(configStr, "toml")
+	c.Assert(err, qt.IsNil)
+	fs := afero.NewMemMapFs()
+
+	_, err = DecodeConfig(fs, cfg)
+	c.Assert(err, qt.Not(qt.IsNil))
+
+}
+
+func newTestConfig() *viper.Viper {
+	cfg := viper.New()
+	cfg.Set("workingDir", filepath.FromSlash("/my/cool/hugoproject"))
+	cfg.Set("contentDir", "content")
+	cfg.Set("dataDir", "data")
+	cfg.Set("resourceDir", "resources")
+	cfg.Set("i18nDir", "i18n")
+	cfg.Set("layoutDir", "layouts")
+	cfg.Set("archetypeDir", "archetypes")
+	cfg.Set("assetDir", "assets")
+
+	return cfg
+}

cache/filecache/filecache_pruner.go

@@ -0,0 +1,127 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"io"
+	"os"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/afero"
+)
+
+// Prune removes expired and unused items from this cache.
+// The last one requires a full build so the cache usage can be tracked.
+// Note that we operate directly on the filesystem here, so this is not
+// thread safe.
+func (c Caches) Prune() (int, error) {
+	counter := 0
+	for k, cache := range c {
+
+		count, err := cache.Prune(false)
+
+		if err != nil {
+			return counter, errors.Wrapf(err, "failed to prune cache %q", k)
+		}
+
+		counter += count
+
+	}
+
+	return counter, nil
+}
+
+// Prune removes expired and unused items from this cache.
+// If force is set, everything will be removed not considering expiry time.
+func (c *Cache) Prune(force bool) (int, error) {
+	if c.pruneAllRootDir != "" {
+		return c.pruneRootDir(force)
+	}
+
+	counter := 0
+
+	err := afero.Walk(c.Fs, "", func(name string, info os.FileInfo, err error) error {
+		if info == nil {
+			return nil
+		}
+
+		name = cleanID(name)
+
+		if info.IsDir() {
+			f, err := c.Fs.Open(name)
+			if err != nil {
+				// This cache dir may not exist.
+				return nil
+			}
+			defer f.Close()
+			_, err = f.Readdirnames(1)
+			if err == io.EOF {
+				// Empty dir.
+				return c.Fs.Remove(name)
+			}
+
+			return nil
+		}
+
+		shouldRemove := force || c.isExpired(info.ModTime())
+
+		if !shouldRemove && len(c.nlocker.seen) > 0 {
+			// Remove it if it's not been touched/used in the last build.
+			_, seen := c.nlocker.seen[name]
+			shouldRemove = !seen
+		}
+
+		if shouldRemove {
+			err := c.Fs.Remove(name)
+			if err == nil {
+				counter++
+			}
+			return err
+		}
+
+		return nil
+	})
+
+	return counter, err
+}
+
+func (c *Cache) pruneRootDir(force bool) (int, error) {
+
+	info, err := c.Fs.Stat(c.pruneAllRootDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return 0, nil
+		}
+		return 0, err
+	}
+
+	if !force && !c.isExpired(info.ModTime()) {
+		return 0, nil
+	}
+
+	counter := 0
+	// Module cache has 0555 directories; make them writable in order to remove content.
+	afero.Walk(c.Fs, c.pruneAllRootDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+		if info.IsDir() {
+			counter++
+			c.Fs.Chmod(path, 0777)
+		}
+		return nil
+	})
+	return 1, c.Fs.RemoveAll(c.pruneAllRootDir)
+
+}

cache/filecache/filecache_pruner_test.go

@@ -0,0 +1,111 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/spf13/afero"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestPrune(t *testing.T) {
+	t.Parallel()
+
+	c := qt.New(t)
+
+	configStr := `
+resourceDir = "myresources"
+contentDir = "content"
+dataDir = "data"
+i18nDir = "i18n"
+layoutDir = "layouts"
+assetDir = "assets"
+archeTypedir = "archetypes"
+
+[caches]
+[caches.getjson]
+maxAge = "200ms"
+dir = "/cache/c"
+[caches.getcsv]
+maxAge = "200ms"
+dir = "/cache/d"
+[caches.assets]
+maxAge = "200ms"
+dir = ":resourceDir/_gen"
+[caches.images]
+maxAge = "200ms"
+dir = ":resourceDir/_gen"
+`
+
+	for _, name := range []string{cacheKeyGetCSV, cacheKeyGetJSON, cacheKeyAssets, cacheKeyImages} {
+		msg := qt.Commentf("cache: %s", name)
+		p := newPathsSpec(t, afero.NewMemMapFs(), configStr)
+		caches, err := NewCaches(p)
+		c.Assert(err, qt.IsNil)
+		cache := caches[name]
+		for i := 0; i < 10; i++ {
+			id := fmt.Sprintf("i%d", i)
+			cache.GetOrCreateBytes(id, func() ([]byte, error) {
+				return []byte("abc"), nil
+			})
+			if i == 4 {
+				// This will expire the first 5
+				time.Sleep(201 * time.Millisecond)
+			}
+		}
+
+		count, err := caches.Prune()
+		c.Assert(err, qt.IsNil)
+		c.Assert(count, qt.Equals, 5, msg)
+
+		for i := 0; i < 10; i++ {
+			id := fmt.Sprintf("i%d", i)
+			v := cache.getString(id)
+			if i < 5 {
+				c.Assert(v, qt.Equals, "")
+			} else {
+				c.Assert(v, qt.Equals, "abc")
+			}
+		}
+
+		caches, err = NewCaches(p)
+		c.Assert(err, qt.IsNil)
+		cache = caches[name]
+		// Touch one and then prune.
+		cache.GetOrCreateBytes("i5", func() ([]byte, error) {
+			return []byte("abc"), nil
+		})
+
+		count, err = caches.Prune()
+		c.Assert(err, qt.IsNil)
+		c.Assert(count, qt.Equals, 4)
+
+		// Now only the i5 should be left.
+		for i := 0; i < 10; i++ {
+			id := fmt.Sprintf("i%d", i)
+			v := cache.getString(id)
+			if i != 5 {
+				c.Assert(v, qt.Equals, "")
+			} else {
+				c.Assert(v, qt.Equals, "abc")
+			}
+		}
+
+	}
+
+}

cache/filecache/filecache_test.go

@@ -0,0 +1,298 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/gohugoio/hugo/langs"
+	"github.com/gohugoio/hugo/modules"
+
+	"github.com/gohugoio/hugo/common/hugio"
+	"github.com/gohugoio/hugo/config"
+	"github.com/gohugoio/hugo/helpers"
+
+	"github.com/gohugoio/hugo/hugofs"
+	"github.com/spf13/afero"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestFileCache(t *testing.T) {
+	t.Parallel()
+	c := qt.New(t)
+
+	tempWorkingDir, err := ioutil.TempDir("", "hugo_filecache_test_work")
+	c.Assert(err, qt.IsNil)
+	defer os.Remove(tempWorkingDir)
+
+	tempCacheDir, err := ioutil.TempDir("", "hugo_filecache_test_cache")
+	c.Assert(err, qt.IsNil)
+	defer os.Remove(tempCacheDir)
+
+	osfs := afero.NewOsFs()
+
+	for _, test := range []struct {
+		cacheDir   string
+		workingDir string
+	}{
+		// Run with same dirs twice to make sure that works.
+		{tempCacheDir, tempWorkingDir},
+		{tempCacheDir, tempWorkingDir},
+	} {
+
+		configStr := `
+workingDir = "WORKING_DIR"
+resourceDir = "resources"
+cacheDir = "CACHEDIR"
+contentDir = "content"
+dataDir = "data"
+i18nDir = "i18n"
+layoutDir = "layouts"
+assetDir = "assets"
+archeTypedir = "archetypes"
+
+[caches]
+[caches.getJSON]
+maxAge = "10h"
+dir = ":cacheDir/c"
+
+`
+
+		winPathSep := "\\\\"
+
+		replacer := strings.NewReplacer("CACHEDIR", test.cacheDir, "WORKING_DIR", test.workingDir)
+
+		configStr = replacer.Replace(configStr)
+		configStr = strings.Replace(configStr, "\\", winPathSep, -1)
+
+		p := newPathsSpec(t, osfs, configStr)
+
+		caches, err := NewCaches(p)
+		c.Assert(err, qt.IsNil)
+
+		cache := caches.Get("GetJSON")
+		c.Assert(cache, qt.Not(qt.IsNil))
+		c.Assert(cache.maxAge.String(), qt.Equals, "10h0m0s")
+
+		bfs, ok := cache.Fs.(*afero.BasePathFs)
+		c.Assert(ok, qt.Equals, true)
+		filename, err := bfs.RealPath("key")
+		c.Assert(err, qt.IsNil)
+		if test.cacheDir != "" {
+			c.Assert(filename, qt.Equals, filepath.Join(test.cacheDir, "c/"+filecacheRootDirname+"/getjson/key"))
+		} else {
+			// Temp dir.
+			c.Assert(filename, qt.Matches, ".*hugo_cache.*"+filecacheRootDirname+".*key")
+		}
+
+		cache = caches.Get("Images")
+		c.Assert(cache, qt.Not(qt.IsNil))
+		c.Assert(cache.maxAge, qt.Equals, time.Duration(-1))
+		bfs, ok = cache.Fs.(*afero.BasePathFs)
+		c.Assert(ok, qt.Equals, true)
+		filename, _ = bfs.RealPath("key")
+		c.Assert(filename, qt.Equals, filepath.FromSlash("_gen/images/key"))
+
+		rf := func(s string) func() (io.ReadCloser, error) {
+			return func() (io.ReadCloser, error) {
+				return struct {
+					io.ReadSeeker
+					io.Closer
+				}{
+					strings.NewReader(s),
+					ioutil.NopCloser(nil),
+				}, nil
+			}
+		}
+
+		bf := func() ([]byte, error) {
+			return []byte("bcd"), nil
+		}
+
+		for _, ca := range []*Cache{caches.ImageCache(), caches.AssetsCache(), caches.GetJSONCache(), caches.GetCSVCache()} {
+			for i := 0; i < 2; i++ {
+				info, r, err := ca.GetOrCreate("a", rf("abc"))
+				c.Assert(err, qt.IsNil)
+				c.Assert(r, qt.Not(qt.IsNil))
+				c.Assert(info.Name, qt.Equals, "a")
+				b, _ := ioutil.ReadAll(r)
+				r.Close()
+				c.Assert(string(b), qt.Equals, "abc")
+
+				info, b, err = ca.GetOrCreateBytes("b", bf)
+				c.Assert(err, qt.IsNil)
+				c.Assert(r, qt.Not(qt.IsNil))
+				c.Assert(info.Name, qt.Equals, "b")
+				c.Assert(string(b), qt.Equals, "bcd")
+
+				_, b, err = ca.GetOrCreateBytes("a", bf)
+				c.Assert(err, qt.IsNil)
+				c.Assert(string(b), qt.Equals, "abc")
+
+				_, r, err = ca.GetOrCreate("a", rf("bcd"))
+				c.Assert(err, qt.IsNil)
+				b, _ = ioutil.ReadAll(r)
+				r.Close()
+				c.Assert(string(b), qt.Equals, "abc")
+			}
+		}
+
+		c.Assert(caches.Get("getJSON"), qt.Not(qt.IsNil))
+
+		info, w, err := caches.ImageCache().WriteCloser("mykey")
+		c.Assert(err, qt.IsNil)
+		c.Assert(info.Name, qt.Equals, "mykey")
+		io.WriteString(w, "Hugo is great!")
+		w.Close()
+		c.Assert(caches.ImageCache().getString("mykey"), qt.Equals, "Hugo is great!")
+
+		info, r, err := caches.ImageCache().Get("mykey")
+		c.Assert(err, qt.IsNil)
+		c.Assert(r, qt.Not(qt.IsNil))
+		c.Assert(info.Name, qt.Equals, "mykey")
+		b, _ := ioutil.ReadAll(r)
+		r.Close()
+		c.Assert(string(b), qt.Equals, "Hugo is great!")
+
+		info, b, err = caches.ImageCache().GetBytes("mykey")
+		c.Assert(err, qt.IsNil)
+		c.Assert(info.Name, qt.Equals, "mykey")
+		c.Assert(string(b), qt.Equals, "Hugo is great!")
+
+	}
+
+}
+
+func TestFileCacheConcurrent(t *testing.T) {
+	t.Parallel()
+
+	c := qt.New(t)
+
+	configStr := `
+resourceDir = "myresources"
+contentDir = "content"
+dataDir = "data"
+i18nDir = "i18n"
+layoutDir = "layouts"
+assetDir = "assets"
+archeTypedir = "archetypes"
+
+[caches]
+[caches.getjson]
+maxAge = "1s"
+dir = "/cache/c"
+
+`
+
+	p := newPathsSpec(t, afero.NewMemMapFs(), configStr)
+
+	caches, err := NewCaches(p)
+	c.Assert(err, qt.IsNil)
+
+	const cacheName = "getjson"
+
+	filenameData := func(i int) (string, string) {
+		data := fmt.Sprintf("data: %d", i)
+		filename := fmt.Sprintf("file%d", i)
+		return filename, data
+	}
+
+	var wg sync.WaitGroup
+
+	for i := 0; i < 50; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			for j := 0; j < 20; j++ {
+				ca := caches.Get(cacheName)
+				c.Assert(ca, qt.Not(qt.IsNil))
+				filename, data := filenameData(i)
+				_, r, err := ca.GetOrCreate(filename, func() (io.ReadCloser, error) {
+					return hugio.ToReadCloser(strings.NewReader(data)), nil
+				})
+				c.Assert(err, qt.IsNil)
+				b, _ := ioutil.ReadAll(r)
+				r.Close()
+				c.Assert(string(b), qt.Equals, data)
+				// Trigger some expiration.
+				time.Sleep(50 * time.Millisecond)
+			}
+		}(i)
+
+	}
+	wg.Wait()
+}
+
+func TestCleanID(t *testing.T) {
+	c := qt.New(t)
+	c.Assert(cleanID(filepath.FromSlash("/a/b//c.txt")), qt.Equals, filepath.FromSlash("a/b/c.txt"))
+	c.Assert(cleanID(filepath.FromSlash("a/b//c.txt")), qt.Equals, filepath.FromSlash("a/b/c.txt"))
+}
+
+func initConfig(fs afero.Fs, cfg config.Provider) error {
+	if _, err := langs.LoadLanguageSettings(cfg, nil); err != nil {
+		return err
+	}
+
+	modConfig, err := modules.DecodeConfig(cfg)
+	if err != nil {
+		return err
+	}
+
+	workingDir := cfg.GetString("workingDir")
+	themesDir := cfg.GetString("themesDir")
+	if !filepath.IsAbs(themesDir) {
+		themesDir = filepath.Join(workingDir, themesDir)
+	}
+	modulesClient := modules.NewClient(modules.ClientConfig{
+		Fs:           fs,
+		WorkingDir:   workingDir,
+		ThemesDir:    themesDir,
+		ModuleConfig: modConfig,
+		IgnoreVendor: true,
+	})
+
+	moduleConfig, err := modulesClient.Collect()
+	if err != nil {
+		return err
+	}
+
+	if err := modules.ApplyProjectConfigDefaults(cfg, moduleConfig.ActiveModules[len(moduleConfig.ActiveModules)-1]); err != nil {
+		return err
+	}
+
+	cfg.Set("allModules", moduleConfig.ActiveModules)
+
+	return nil
+}
+
+func newPathsSpec(t *testing.T, fs afero.Fs, configStr string) *helpers.PathSpec {
+	c := qt.New(t)
+	cfg, err := config.FromConfigString(configStr, "toml")
+	c.Assert(err, qt.IsNil)
+	initConfig(fs, cfg)
+	p, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg, nil)
+	c.Assert(err, qt.IsNil)
+	return p
+
+}

cache/namedmemcache/named_cache.go

@@ -0,0 +1,79 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package namedmemcache provides a memory cache with a named lock. This is suitable
+// for situations where creating the cached resource can be time consuming or otherwise
+// resource hungry, or in situations where a "once only per key" is a requirement.
+package namedmemcache
+
+import (
+	"sync"
+
+	"github.com/BurntSushi/locker"
+)
+
+// Cache holds the cached values.
+type Cache struct {
+	nlocker *locker.Locker
+	cache   map[string]cacheEntry
+	mu      sync.RWMutex
+}
+
+type cacheEntry struct {
+	value interface{}
+	err   error
+}
+
+// New creates a new cache.
+func New() *Cache {
+	return &Cache{
+		nlocker: locker.NewLocker(),
+		cache:   make(map[string]cacheEntry),
+	}
+}
+
+// Clear clears the cache state.
+func (c *Cache) Clear() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.cache = make(map[string]cacheEntry)
+	c.nlocker = locker.NewLocker()
+
+}
+
+// GetOrCreate tries to get the value with the given cache key, if not found
+// create will be called and cached.
+// This method is thread safe. It also guarantees that the create func for a given
+// key is invoced only once for this cache.
+func (c *Cache) GetOrCreate(key string, create func() (interface{}, error)) (interface{}, error) {
+	c.mu.RLock()
+	entry, found := c.cache[key]
+	c.mu.RUnlock()
+
+	if found {
+		return entry.value, entry.err
+	}
+
+	c.nlocker.Lock(key)
+	defer c.nlocker.Unlock(key)
+
+	// Create it.
+	value, err := create()
+
+	c.mu.Lock()
+	c.cache[key] = cacheEntry{value: value, err: err}
+	c.mu.Unlock()
+
+	return value, err
+}

cache/namedmemcache/named_cache_test.go

@@ -0,0 +1,80 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package namedmemcache
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestNamedCache(t *testing.T) {
+	t.Parallel()
+	c := qt.New(t)
+
+	cache := New()
+
+	counter := 0
+	create := func() (interface{}, error) {
+		counter++
+		return counter, nil
+	}
+
+	for i := 0; i < 5; i++ {
+		v1, err := cache.GetOrCreate("a1", create)
+		c.Assert(err, qt.IsNil)
+		c.Assert(v1, qt.Equals, 1)
+		v2, err := cache.GetOrCreate("a2", create)
+		c.Assert(err, qt.IsNil)
+		c.Assert(v2, qt.Equals, 2)
+	}
+
+	cache.Clear()
+
+	v3, err := cache.GetOrCreate("a2", create)
+	c.Assert(err, qt.IsNil)
+	c.Assert(v3, qt.Equals, 3)
+}
+
+func TestNamedCacheConcurrent(t *testing.T) {
+	t.Parallel()
+
+	c := qt.New(t)
+
+	var wg sync.WaitGroup
+
+	cache := New()
+
+	create := func(i int) func() (interface{}, error) {
+		return func() (interface{}, error) {
+			return i, nil
+		}
+	}
+
+	for i := 0; i < 10; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := 0; j < 100; j++ {
+				id := fmt.Sprintf("id%d", j)
+				v, err := cache.GetOrCreate(id, create(j))
+				c.Assert(err, qt.IsNil)
+				c.Assert(v, qt.Equals, j)
+			}
+		}()
+	}
+	wg.Wait()
+}

cache/partitioned_lazy_cache.go

@@ -24,34 +24,52 @@ type Partition struct {
 	Load func() (map[string]interface{}, error)
 }
 
-type lazyPartition struct {
+// Lazy represents a lazily loaded cache.
+type Lazy struct {
 	initSync sync.Once
+	initErr  error
 	cache    map[string]interface{}
 	load     func() (map[string]interface{}, error)
 }
 
-func (l *lazyPartition) init() error {
-	var err error
+// NewLazy creates a lazy cache with the given load func.
+func NewLazy(load func() (map[string]interface{}, error)) *Lazy {
+	return &Lazy{load: load}
+}
+
+func (l *Lazy) init() error {
 	l.initSync.Do(func() {
-		var c map[string]interface{}
-		c, err = l.load()
+		c, err := l.load()
 		l.cache = c
+		l.initErr = err
+
 	})
 
-	return err
+	return l.initErr
+}
+
+// Get initializes the cache if not already initialized, then looks up the
+// given key.
+func (l *Lazy) Get(key string) (interface{}, bool, error) {
+	l.init()
+	if l.initErr != nil {
+		return nil, false, l.initErr
+	}
+	v, found := l.cache[key]
+	return v, found, nil
 }
 
 // PartitionedLazyCache is a lazily loaded cache paritioned by a supplied string key.
 type PartitionedLazyCache struct {
-	partitions map[string]*lazyPartition
+	partitions map[string]*Lazy
 }
 
 // NewPartitionedLazyCache creates a new NewPartitionedLazyCache with the supplied
 // partitions.
 func NewPartitionedLazyCache(partitions ...Partition) *PartitionedLazyCache {
-	lazyPartitions := make(map[string]*lazyPartition, len(partitions))
+	lazyPartitions := make(map[string]*Lazy, len(partitions))
 	for _, partition := range partitions {
-		lazyPartitions[partition.Key] = &lazyPartition{load: partition.Load}
+		lazyPartitions[partition.Key] = NewLazy(partition.Load)
 	}
 	cache := &PartitionedLazyCache{partitions: lazyPartitions}
 
@@ -67,11 +85,12 @@ func (c *PartitionedLazyCache) Get(partition, key string) (interface{}, error) {
 		return nil, nil
 	}
 
-	if err := p.init(); err != nil {
+	v, found, err := p.Get(key)
+	if err != nil {
 		return nil, err
 	}
 
-	if v, found := p.cache[key]; found {
+	if found {
 		return v, nil
 	}
 

cache/partitioned_lazy_cache_test.go

@@ -18,13 +18,13 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/stretchr/testify/require"
+	qt "github.com/frankban/quicktest"
 )
 
 func TestNewPartitionedLazyCache(t *testing.T) {
 	t.Parallel()
 
-	assert := require.New(t)
+	c := qt.New(t)
 
 	p1 := Partition{
 		Key: "p1",
@@ -51,28 +51,28 @@ func TestNewPartitionedLazyCache(t *testing.T) {
 	cache := NewPartitionedLazyCache(p1, p2)
 
 	v, err := cache.Get("p1", "p1_1")
-	assert.NoError(err)
-	assert.Equal("p1v1", v)
+	c.Assert(err, qt.IsNil)
+	c.Assert(v, qt.Equals, "p1v1")
 
 	v, err = cache.Get("p1", "p2_1")
-	assert.NoError(err)
-	assert.Nil(v)
+	c.Assert(err, qt.IsNil)
+	c.Assert(v, qt.IsNil)
 
 	v, err = cache.Get("p1", "p1_nil")
-	assert.NoError(err)
-	assert.Nil(v)
+	c.Assert(err, qt.IsNil)
+	c.Assert(v, qt.IsNil)
 
 	v, err = cache.Get("p2", "p2_3")
-	assert.NoError(err)
-	assert.Equal("p2v3", v)
+	c.Assert(err, qt.IsNil)
+	c.Assert(v, qt.Equals, "p2v3")
 
 	v, err = cache.Get("doesnotexist", "p1_1")
-	assert.NoError(err)
-	assert.Nil(v)
+	c.Assert(err, qt.IsNil)
+	c.Assert(v, qt.IsNil)
 
 	v, err = cache.Get("p1", "doesnotexist")
-	assert.NoError(err)
-	assert.Nil(v)
+	c.Assert(err, qt.IsNil)
+	c.Assert(v, qt.IsNil)
 
 	errorP := Partition{
 		Key: "p3",
@@ -84,18 +84,18 @@ func TestNewPartitionedLazyCache(t *testing.T) {
 	cache = NewPartitionedLazyCache(errorP)
 
 	v, err = cache.Get("p1", "doesnotexist")
-	assert.NoError(err)
-	assert.Nil(v)
+	c.Assert(err, qt.IsNil)
+	c.Assert(v, qt.IsNil)
 
 	_, err = cache.Get("p3", "doesnotexist")
-	assert.Error(err)
+	c.Assert(err, qt.Not(qt.IsNil))
 
 }
 
 func TestConcurrentPartitionedLazyCache(t *testing.T) {
 	t.Parallel()
 
-	assert := require.New(t)
+	c := qt.New(t)
 
 	var wg sync.WaitGroup
 
@@ -129,8 +129,8 @@ func TestConcurrentPartitionedLazyCache(t *testing.T) {
 			defer wg.Done()
 			for j := 0; j < 10; j++ {
 				v, err := cache.Get("p1", "p1_1")
-				assert.NoError(err)
-				assert.Equal("p1v1", v)
+				c.Assert(err, qt.IsNil)
+				c.Assert(v, qt.Equals, "p1v1")
 			}
 		}()
 	}

codegen/methods.go

@@ -0,0 +1,548 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+// Some functions in this file (see comments) is based on the Go source code,
+// copyright The Go Authors and  governed by a BSD-style license.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package codegen contains helpers for code generation.
+package codegen
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"os"
+	"path"
+	"path/filepath"
+	"reflect"
+	"regexp"
+	"sort"
+	"strings"
+	"sync"
+)
+
+// Make room for insertions
+const weightWidth = 1000
+
+// NewInspector creates a new Inspector given a source root.
+func NewInspector(root string) *Inspector {
+	return &Inspector{ProjectRootDir: root}
+}
+
+// Inspector provides methods to help code generation. It uses a combination
+// of reflection and source code AST to do the heavy lifting.
+type Inspector struct {
+	ProjectRootDir string
+
+	init sync.Once
+
+	// Determines method order. Go's reflect sorts lexicographically, so
+	// we must parse the source to preserve this order.
+	methodWeight map[string]map[string]int
+}
+
+// MethodsFromTypes create a method set from the include slice, excluding any
+// method in exclude.
+func (c *Inspector) MethodsFromTypes(include []reflect.Type, exclude []reflect.Type) Methods {
+	c.parseSource()
+
+	var methods Methods
+
+	var excludes = make(map[string]bool)
+
+	if len(exclude) > 0 {
+		for _, m := range c.MethodsFromTypes(exclude, nil) {
+			excludes[m.Name] = true
+		}
+	}
+
+	// There may be overlapping interfaces in types. Do a simple check for now.
+	seen := make(map[string]bool)
+
+	nameAndPackage := func(t reflect.Type) (string, string) {
+		var name, pkg string
+
+		isPointer := t.Kind() == reflect.Ptr
+
+		if isPointer {
+			t = t.Elem()
+		}
+
+		pkgPrefix := ""
+		if pkgPath := t.PkgPath(); pkgPath != "" {
+			pkgPath = strings.TrimSuffix(pkgPath, "/")
+			_, shortPath := path.Split(pkgPath)
+			pkgPrefix = shortPath + "."
+			pkg = pkgPath
+		}
+
+		name = t.Name()
+		if name == "" {
+			// interface{}
+			name = t.String()
+		}
+
+		if isPointer {
+			pkgPrefix = "*" + pkgPrefix
+		}
+
+		name = pkgPrefix + name
+
+		return name, pkg
+
+	}
+
+	for _, t := range include {
+
+		for i := 0; i < t.NumMethod(); i++ {
+
+			m := t.Method(i)
+			if excludes[m.Name] || seen[m.Name] {
+				continue
+			}
+
+			seen[m.Name] = true
+
+			if m.PkgPath != "" {
+				// Not exported
+				continue
+			}
+
+			numIn := m.Type.NumIn()
+
+			ownerName, _ := nameAndPackage(t)
+
+			method := Method{Owner: t, OwnerName: ownerName, Name: m.Name}
+
+			for i := 0; i < numIn; i++ {
+				in := m.Type.In(i)
+
+				name, pkg := nameAndPackage(in)
+
+				if pkg != "" {
+					method.Imports = append(method.Imports, pkg)
+				}
+
+				method.In = append(method.In, name)
+			}
+
+			numOut := m.Type.NumOut()
+
+			if numOut > 0 {
+				for i := 0; i < numOut; i++ {
+					out := m.Type.Out(i)
+					name, pkg := nameAndPackage(out)
+
+					if pkg != "" {
+						method.Imports = append(method.Imports, pkg)
+					}
+
+					method.Out = append(method.Out, name)
+				}
+			}
+
+			methods = append(methods, method)
+		}
+
+	}
+
+	sort.SliceStable(methods, func(i, j int) bool {
+		mi, mj := methods[i], methods[j]
+
+		wi := c.methodWeight[mi.OwnerName][mi.Name]
+		wj := c.methodWeight[mj.OwnerName][mj.Name]
+
+		if wi == wj {
+			return mi.Name < mj.Name
+		}
+
+		return wi < wj
+
+	})
+
+	return methods
+
+}
+
+func (c *Inspector) parseSource() {
+	c.init.Do(func() {
+
+		if !strings.Contains(c.ProjectRootDir, "hugo") {
+			panic("dir must be set to the Hugo root")
+		}
+
+		c.methodWeight = make(map[string]map[string]int)
+		dirExcludes := regexp.MustCompile("docs|examples")
+		fileExcludes := regexp.MustCompile("autogen")
+		var filenames []string
+
+		filepath.Walk(c.ProjectRootDir, func(path string, info os.FileInfo, err error) error {
+			if info.IsDir() {
+				if dirExcludes.MatchString(info.Name()) {
+					return filepath.SkipDir
+				}
+			}
+
+			if !strings.HasSuffix(path, ".go") || fileExcludes.MatchString(path) {
+				return nil
+			}
+
+			filenames = append(filenames, path)
+
+			return nil
+
+		})
+
+		for _, filename := range filenames {
+
+			pkg := c.packageFromPath(filename)
+
+			fset := token.NewFileSet()
+			node, err := parser.ParseFile(fset, filename, nil, parser.ParseComments)
+			if err != nil {
+				panic(err)
+			}
+
+			ast.Inspect(node, func(n ast.Node) bool {
+				switch t := n.(type) {
+				case *ast.TypeSpec:
+					if t.Name.IsExported() {
+						switch it := t.Type.(type) {
+						case *ast.InterfaceType:
+							iface := pkg + "." + t.Name.Name
+							methodNames := collectMethodsRecursive(pkg, it.Methods.List)
+							weights := make(map[string]int)
+							weight := weightWidth
+							for _, name := range methodNames {
+								weights[name] = weight
+								weight += weightWidth
+							}
+							c.methodWeight[iface] = weights
+						}
+					}
+
+				}
+				return true
+			})
+
+		}
+
+		// Complement
+		for _, v1 := range c.methodWeight {
+			for k2, w := range v1 {
+				if v, found := c.methodWeight[k2]; found {
+					for k3, v3 := range v {
+						v1[k3] = (v3 / weightWidth) + w
+					}
+				}
+			}
+		}
+
+	})
+}
+
+func (c *Inspector) packageFromPath(p string) string {
+	p = filepath.ToSlash(p)
+	base := path.Base(p)
+	if !strings.Contains(base, ".") {
+		return base
+	}
+	return path.Base(strings.TrimSuffix(p, base))
+}
+
+// Method holds enough information about it to recreate it.
+type Method struct {
+	// The interface we extracted this method from.
+	Owner reflect.Type
+
+	// String version of the above, on the form PACKAGE.NAME, e.g.
+	// page.Page
+	OwnerName string
+
+	// Method name.
+	Name string
+
+	// Imports needed to satisfy the method signature.
+	Imports []string
+
+	// Argument types, including any package prefix, e.g. string, int, interface{},
+	// net.Url
+	In []string
+
+	// Return types.
+	Out []string
+}
+
+// Declaration creates a method declaration (without any body) for the given receiver.
+func (m Method) Declaration(receiver string) string {
+	return fmt.Sprintf("func (%s %s) %s%s %s", receiverShort(receiver), receiver, m.Name, m.inStr(), m.outStr())
+}
+
+// DeclarationNamed creates a method declaration (without any body) for the given receiver
+// with named return values.
+func (m Method) DeclarationNamed(receiver string) string {
+	return fmt.Sprintf("func (%s %s) %s%s %s", receiverShort(receiver), receiver, m.Name, m.inStr(), m.outStrNamed())
+}
+
+// Delegate creates a delegate call string.
+func (m Method) Delegate(receiver, delegate string) string {
+	ret := ""
+	if len(m.Out) > 0 {
+		ret = "return "
+	}
+	return fmt.Sprintf("%s%s.%s.%s%s", ret, receiverShort(receiver), delegate, m.Name, m.inOutStr())
+}
+
+func (m Method) String() string {
+	return m.Name + m.inStr() + " " + m.outStr() + "\n"
+}
+
+func (m Method) inOutStr() string {
+	if len(m.In) == 0 {
+		return "()"
+	}
+
+	args := make([]string, len(m.In))
+	for i := 0; i < len(args); i++ {
+		args[i] = fmt.Sprintf("arg%d", i)
+	}
+	return "(" + strings.Join(args, ", ") + ")"
+}
+
+func (m Method) inStr() string {
+	if len(m.In) == 0 {
+		return "()"
+	}
+
+	args := make([]string, len(m.In))
+	for i := 0; i < len(args); i++ {
+		args[i] = fmt.Sprintf("arg%d %s", i, m.In[i])
+	}
+	return "(" + strings.Join(args, ", ") + ")"
+}
+
+func (m Method) outStr() string {
+	if len(m.Out) == 0 {
+		return ""
+	}
+	if len(m.Out) == 1 {
+		return m.Out[0]
+	}
+
+	return "(" + strings.Join(m.Out, ", ") + ")"
+}
+
+func (m Method) outStrNamed() string {
+	if len(m.Out) == 0 {
+		return ""
+	}
+
+	outs := make([]string, len(m.Out))
+	for i := 0; i < len(outs); i++ {
+		outs[i] = fmt.Sprintf("o%d %s", i, m.Out[i])
+	}
+
+	return "(" + strings.Join(outs, ", ") + ")"
+}
+
+// Methods represents a list of methods for one or more interfaces.
+// The order matches the defined order in their source file(s).
+type Methods []Method
+
+// Imports returns a sorted list of package imports needed to satisfy the
+// signatures of all methods.
+func (m Methods) Imports() []string {
+	var pkgImports []string
+	for _, method := range m {
+		pkgImports = append(pkgImports, method.Imports...)
+	}
+	if len(pkgImports) > 0 {
+		pkgImports = uniqueNonEmptyStrings(pkgImports)
+		sort.Strings(pkgImports)
+	}
+	return pkgImports
+}
+
+// ToMarshalJSON creates a MarshalJSON method for these methods. Any method name
+// matchin any of the regexps in excludes will be ignored.
+func (m Methods) ToMarshalJSON(receiver, pkgPath string, excludes ...string) (string, []string) {
+	var sb strings.Builder
+
+	r := receiverShort(receiver)
+	what := firstToUpper(trimAsterisk(receiver))
+	pgkName := path.Base(pkgPath)
+
+	fmt.Fprintf(&sb, "func Marshal%sToJSON(%s %s) ([]byte, error) {\n", what, r, receiver)
+
+	var methods Methods
+	var excludeRes = make([]*regexp.Regexp, len(excludes))
+
+	for i, exclude := range excludes {
+		excludeRes[i] = regexp.MustCompile(exclude)
+	}
+
+	for _, method := range m {
+		// Exclude methods with arguments and incompatible return values
+		if len(method.In) > 0 || len(method.Out) == 0 || len(method.Out) > 2 {
+			continue
+		}
+
+		if len(method.Out) == 2 {
+			if method.Out[1] != "error" {
+				continue
+			}
+		}
+
+		for _, re := range excludeRes {
+			if re.MatchString(method.Name) {
+				continue
+			}
+		}
+
+		methods = append(methods, method)
+	}
+
+	for _, method := range methods {
+		varn := varName(method.Name)
+		if len(method.Out) == 1 {
+			fmt.Fprintf(&sb, "\t%s := %s.%s()\n", varn, r, method.Name)
+		} else {
+			fmt.Fprintf(&sb, "\t%s, err := %s.%s()\n", varn, r, method.Name)
+			fmt.Fprint(&sb, "\tif err != nil {\n\t\treturn nil, err\n\t}\n")
+		}
+	}
+
+	fmt.Fprint(&sb, "\n\ts := struct {\n")
+
+	for _, method := range methods {
+		fmt.Fprintf(&sb, "\t\t%s %s\n", method.Name, typeName(method.Out[0], pgkName))
+	}
+
+	fmt.Fprint(&sb, "\n\t}{\n")
+
+	for _, method := range methods {
+		varn := varName(method.Name)
+		fmt.Fprintf(&sb, "\t\t%s: %s,\n", method.Name, varn)
+	}
+
+	fmt.Fprint(&sb, "\n\t}\n\n")
+	fmt.Fprint(&sb, "\treturn json.Marshal(&s)\n}")
+
+	pkgImports := append(methods.Imports(), "encoding/json")
+
+	if pkgPath != "" {
+		// Exclude self
+		for i, pkgImp := range pkgImports {
+			if pkgImp == pkgPath {
+				pkgImports = append(pkgImports[:i], pkgImports[i+1:]...)
+			}
+		}
+	}
+
+	return sb.String(), pkgImports
+
+}
+
+func collectMethodsRecursive(pkg string, f []*ast.Field) []string {
+	var methodNames []string
+	for _, m := range f {
+		if m.Names != nil {
+			methodNames = append(methodNames, m.Names[0].Name)
+			continue
+		}
+
+		if ident, ok := m.Type.(*ast.Ident); ok && ident.Obj != nil {
+			// Embedded interface
+			methodNames = append(
+				methodNames,
+				collectMethodsRecursive(
+					pkg,
+					ident.Obj.Decl.(*ast.TypeSpec).Type.(*ast.InterfaceType).Methods.List)...)
+		} else {
+			// Embedded, but in a different file/package. Return the
+			// package.Name and deal with that later.
+			name := packageName(m.Type)
+			if !strings.Contains(name, ".") {
+				// Assume current package
+				name = pkg + "." + name
+			}
+			methodNames = append(methodNames, name)
+		}
+	}
+
+	return methodNames
+
+}
+
+func firstToLower(name string) string {
+	return strings.ToLower(name[:1]) + name[1:]
+}
+
+func firstToUpper(name string) string {
+	return strings.ToUpper(name[:1]) + name[1:]
+}
+
+func packageName(e ast.Expr) string {
+	switch tp := e.(type) {
+	case *ast.Ident:
+		return tp.Name
+	case *ast.SelectorExpr:
+		return fmt.Sprintf("%s.%s", packageName(tp.X), packageName(tp.Sel))
+	}
+	return ""
+}
+
+func receiverShort(receiver string) string {
+	return strings.ToLower(trimAsterisk(receiver))[:1]
+}
+
+func trimAsterisk(name string) string {
+	return strings.TrimPrefix(name, "*")
+}
+
+func typeName(name, pkg string) string {
+	return strings.TrimPrefix(name, pkg+".")
+}
+
+func uniqueNonEmptyStrings(s []string) []string {
+	var unique []string
+	set := map[string]interface{}{}
+	for _, val := range s {
+		if val == "" {
+			continue
+		}
+		if _, ok := set[val]; !ok {
+			unique = append(unique, val)
+			set[val] = val
+		}
+	}
+	return unique
+}
+
+func varName(name string) string {
+	name = firstToLower(name)
+
+	// Adjust some reserved keywords, see https://golang.org/ref/spec#Keywords
+	switch name {
+	case "type":
+		name = "typ"
+	case "package":
+		name = "pkg"
+		// Not reserved, but syntax highlighters has it as a keyword.
+	case "len":
+		name = "length"
+	}
+
+	return name
+
+}

codegen/methods2_test.go

@@ -0,0 +1,20 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package codegen
+
+type IEmbed interface {
+	MethodEmbed3(s string) string
+	MethodEmbed1() string
+	MethodEmbed2()
+}

codegen/methods_test.go

@@ -0,0 +1,100 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package codegen
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"reflect"
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+	"github.com/gohugoio/hugo/common/herrors"
+)
+
+func TestMethods(t *testing.T) {
+
+	var (
+		zeroIE     = reflect.TypeOf((*IEmbed)(nil)).Elem()
+		zeroIEOnly = reflect.TypeOf((*IEOnly)(nil)).Elem()
+		zeroI      = reflect.TypeOf((*I)(nil)).Elem()
+	)
+
+	dir, _ := os.Getwd()
+	insp := NewInspector(dir)
+
+	t.Run("MethodsFromTypes", func(t *testing.T) {
+		c := qt.New(t)
+
+		methods := insp.MethodsFromTypes([]reflect.Type{zeroI}, nil)
+
+		methodsStr := fmt.Sprint(methods)
+
+		c.Assert(methodsStr, qt.Contains, "Method1(arg0 herrors.ErrorContext)")
+		c.Assert(methodsStr, qt.Contains, "Method7() interface {}")
+		c.Assert(methodsStr, qt.Contains, "Method0() string\n Method4() string")
+		c.Assert(methodsStr, qt.Contains, "MethodEmbed3(arg0 string) string\n MethodEmbed1() string")
+
+		c.Assert(methods.Imports(), qt.Contains, "github.com/gohugoio/hugo/common/herrors")
+	})
+
+	t.Run("EmbedOnly", func(t *testing.T) {
+		c := qt.New(t)
+
+		methods := insp.MethodsFromTypes([]reflect.Type{zeroIEOnly}, nil)
+
+		methodsStr := fmt.Sprint(methods)
+
+		c.Assert(methodsStr, qt.Contains, "MethodEmbed3(arg0 string) string")
+
+	})
+
+	t.Run("ToMarshalJSON", func(t *testing.T) {
+		c := qt.New(t)
+
+		m, pkg := insp.MethodsFromTypes(
+			[]reflect.Type{zeroI},
+			[]reflect.Type{zeroIE}).ToMarshalJSON("*page", "page")
+
+		c.Assert(m, qt.Contains, "method6 := p.Method6()")
+		c.Assert(m, qt.Contains, "Method0: method0,")
+		c.Assert(m, qt.Contains, "return json.Marshal(&s)")
+
+		c.Assert(pkg, qt.Contains, "github.com/gohugoio/hugo/common/herrors")
+		c.Assert(pkg, qt.Contains, "encoding/json")
+
+		fmt.Println(pkg)
+
+	})
+
+}
+
+type I interface {
+	IEmbed
+	Method0() string
+	Method4() string
+	Method1(myerr herrors.ErrorContext)
+	Method3(myint int, mystring string)
+	Method5() (string, error)
+	Method6() *net.IP
+	Method7() interface{}
+	Method8() herrors.ErrorContext
+	method2()
+	method9() os.FileInfo
+}
+
+type IEOnly interface {
+	IEmbed
+}

commands/benchmark.go

@@ -1,115 +0,0 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package commands
-
-import (
-	"os"
-	"runtime"
-	"runtime/pprof"
-	"time"
-
-	"github.com/spf13/cobra"
-	jww "github.com/spf13/jwalterweatherman"
-)
-
-type benchmarkCmd struct {
-	benchmarkTimes int
-	cpuProfileFile string
-	memProfileFile string
-
-	*baseBuilderCmd
-}
-
-func (b *commandsBuilder) newBenchmarkCmd() *benchmarkCmd {
-	cmd := &cobra.Command{
-		Use:   "benchmark",
-		Short: "Benchmark Hugo by building a site a number of times.",
-		Long: `Hugo can build a site many times over and analyze the running process
-creating a benchmark.`,
-	}
-
-	c := &benchmarkCmd{baseBuilderCmd: b.newBuilderCmd(cmd)}
-
-	cmd.Flags().StringVar(&c.cpuProfileFile, "cpuprofile", "", "path/filename for the CPU profile file")
-	cmd.Flags().StringVar(&c.memProfileFile, "memprofile", "", "path/filename for the memory profile file")
-	cmd.Flags().IntVarP(&c.benchmarkTimes, "count", "n", 13, "number of times to build the site")
-	cmd.Flags().Bool("renderToMemory", false, "render to memory (only useful for benchmark testing)")
-
-	cmd.RunE = c.benchmark
-
-	return c
-}
-
-func (c *benchmarkCmd) benchmark(cmd *cobra.Command, args []string) error {
-	cfgInit := func(c *commandeer) error {
-		return nil
-	}
-
-	comm, err := initializeConfig(false, &c.hugoBuilderCommon, c, cfgInit)
-	if err != nil {
-		return err
-	}
-
-	var memProf *os.File
-	if c.memProfileFile != "" {
-		memProf, err = os.Create(c.memProfileFile)
-		if err != nil {
-			return err
-		}
-	}
-
-	var cpuProf *os.File
-	if c.cpuProfileFile != "" {
-		cpuProf, err = os.Create(c.cpuProfileFile)
-		if err != nil {
-			return err
-		}
-	}
-
-	var memStats runtime.MemStats
-	runtime.ReadMemStats(&memStats)
-	memAllocated := memStats.TotalAlloc
-	mallocs := memStats.Mallocs
-	if cpuProf != nil {
-		pprof.StartCPUProfile(cpuProf)
-	}
-
-	t := time.Now()
-	for i := 0; i < c.benchmarkTimes; i++ {
-		if err = comm.resetAndBuildSites(); err != nil {
-			return err
-		}
-	}
-	totalTime := time.Since(t)
-
-	if memProf != nil {
-		pprof.WriteHeapProfile(memProf)
-		memProf.Close()
-	}
-	if cpuProf != nil {
-		pprof.StopCPUProfile()
-		cpuProf.Close()
-	}
-
-	runtime.ReadMemStats(&memStats)
-	totalMemAllocated := memStats.TotalAlloc - memAllocated
-	totalMallocs := memStats.Mallocs - mallocs
-
-	jww.FEEDBACK.Println()
-	jww.FEEDBACK.Printf("Average time per operation: %vms\n", int(1000*totalTime.Seconds()/float64(c.benchmarkTimes)))
-	jww.FEEDBACK.Printf("Average memory allocated per operation: %vkB\n", totalMemAllocated/uint64(c.benchmarkTimes)/1024)
-	jww.FEEDBACK.Printf("Average allocations per operation: %v\n", totalMallocs/uint64(c.benchmarkTimes))
-
-	return nil
-}

commands/commandeer.go

@@ -1,4 +1,4 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,58 +14,129 @@
 package commands
 
 import (
+	"bytes"
+	"errors"
+	"sync"
+
+	"golang.org/x/sync/semaphore"
+
+	"github.com/gohugoio/hugo/modules"
+
+	"io/ioutil"
+
+	"github.com/gohugoio/hugo/common/herrors"
+	"github.com/gohugoio/hugo/common/hugo"
+
+	jww "github.com/spf13/jwalterweatherman"
+
 	"os"
 	"path/filepath"
-	"sync"
+	"regexp"
 	"time"
 
+	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/config"
 
 	"github.com/spf13/cobra"
 
-	"github.com/gohugoio/hugo/utils"
-
-	"github.com/spf13/afero"
-
 	"github.com/gohugoio/hugo/hugolib"
+	"github.com/spf13/afero"
 
 	"github.com/bep/debounce"
 	"github.com/gohugoio/hugo/common/types"
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
-	src "github.com/gohugoio/hugo/source"
+	"github.com/gohugoio/hugo/langs"
 )
 
-type commandeer struct {
+type commandeerHugoState struct {
 	*deps.DepsCfg
+	hugoSites *hugolib.HugoSites
+	fsCreate  sync.Once
+	created   chan struct{}
+}
 
-	hugo *hugolib.HugoSites
+type commandeer struct {
+	*commandeerHugoState
+
+	logger *loggers.Logger
+
+	// Currently only set when in "fast render mode". But it seems to
+	// be fast enough that we could maybe just add it for all server modes.
+	changeDetector *fileChangeDetector
+
+	// We need to reuse this on server rebuilds.
+	destinationFs afero.Fs
 
 	h    *hugoBuilderCommon
 	ftch flagsToConfigHandler
 
-	pathSpec    *helpers.PathSpec
 	visitedURLs *types.EvictingStringQueue
 
-	staticDirsConfig []*src.Dirs
-
-	// We watch these for changes.
-	configFiles []string
-
 	doWithCommandeer func(c *commandeer) error
 
-	// We can do this only once.
-	fsCreate sync.Once
+	// We watch these for changes.
+	configFiles []string
 
 	// Used in cases where we get flooded with events in server mode.
 	debounce func(f func())
 
 	serverPorts         []int
 	languagesConfigured bool
-	languages           helpers.Languages
+	languages           langs.Languages
+	doLiveReload        bool
+	fastRenderMode      bool
+	showErrorInBrowser  bool
 
 	configured bool
+	paused     bool
+
+	fullRebuildSem *semaphore.Weighted
+
+	// Any error from the last build.
+	buildErr error
+}
+
+func newCommandeerHugoState() *commandeerHugoState {
+	return &commandeerHugoState{
+		created: make(chan struct{}),
+	}
+}
+
+func (c *commandeerHugoState) hugo() *hugolib.HugoSites {
+	<-c.created
+	return c.hugoSites
+}
+
+func (c *commandeer) errCount() int {
+	return int(c.logger.ErrorCounter.Count())
+}
+
+func (c *commandeer) getErrorWithContext() interface{} {
+	errCount := c.errCount()
+
+	if errCount == 0 {
+		return nil
+	}
+
+	m := make(map[string]interface{})
+
+	m["Error"] = errors.New(removeErrorPrefixFromLog(c.logger.Errors()))
+	m["Version"] = hugo.BuildVersionString()
+
+	fe := herrors.UnwrapErrorWithFileContext(c.buildErr)
+	if fe != nil {
+		m["File"] = fe
+	}
+
+	if c.h.verbose {
+		var b bytes.Buffer
+		herrors.FprintStackTrace(&b, c.buildErr)
+		m["StackTrace"] = b.String()
+	}
+
+	return m
 }
 
 func (c *commandeer) Set(key string, value interface{}) {
@@ -75,57 +146,111 @@ func (c *commandeer) Set(key string, value interface{}) {
 	c.Cfg.Set(key, value)
 }
 
-// PathSpec lazily creates a new PathSpec, as all the paths must
-// be configured before it is created.
-func (c *commandeer) PathSpec() *helpers.PathSpec {
-	c.configured = true
-	return c.pathSpec
-}
-
 func (c *commandeer) initFs(fs *hugofs.Fs) error {
+	c.destinationFs = fs.Destination
 	c.DepsCfg.Fs = fs
-	ps, err := helpers.NewPathSpec(fs, c.Cfg)
-	if err != nil {
-		return err
-	}
-	c.pathSpec = ps
-
-	dirsConfig, err := c.createStaticDirsConfig()
-	if err != nil {
-		return err
-	}
-	c.staticDirsConfig = dirsConfig
 
 	return nil
 }
 
-func newCommandeer(running bool, h *hugoBuilderCommon, f flagsToConfigHandler, doWithCommandeer func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) {
+func newCommandeer(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f flagsToConfigHandler, doWithCommandeer func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) {
 
 	var rebuildDebouncer func(f func())
 	if running {
 		// The time value used is tested with mass content replacements in a fairly big Hugo site.
 		// It is better to wait for some seconds in those cases rather than get flooded
 		// with rebuilds.
-		rebuildDebouncer, _, _ = debounce.New(4 * time.Second)
+		rebuildDebouncer = debounce.New(4 * time.Second)
 	}
 
 	c := &commandeer{
-		h:                h,
-		ftch:             f,
-		doWithCommandeer: doWithCommandeer,
-		visitedURLs:      types.NewEvictingStringQueue(10),
-		debounce:         rebuildDebouncer,
+		h:                   h,
+		ftch:                f,
+		commandeerHugoState: newCommandeerHugoState(),
+		doWithCommandeer:    doWithCommandeer,
+		visitedURLs:         types.NewEvictingStringQueue(10),
+		debounce:            rebuildDebouncer,
+		fullRebuildSem:      semaphore.NewWeighted(1),
+		// This will be replaced later, but we need something to log to before the configuration is read.
+		logger: loggers.NewLogger(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, running),
 	}
 
-	return c, c.loadConfig(running)
+	return c, c.loadConfig(mustHaveConfigFile, running)
 }
 
-func (c *commandeer) loadConfig(running bool) error {
+type fileChangeDetector struct {
+	sync.Mutex
+	current map[string]string
+	prev    map[string]string
+
+	irrelevantRe *regexp.Regexp
+}
+
+func (f *fileChangeDetector) OnFileClose(name, md5sum string) {
+	f.Lock()
+	defer f.Unlock()
+	f.current[name] = md5sum
+}
+
+func (f *fileChangeDetector) changed() []string {
+	if f == nil {
+		return nil
+	}
+	f.Lock()
+	defer f.Unlock()
+	var c []string
+	for k, v := range f.current {
+		vv, found := f.prev[k]
+		if !found || v != vv {
+			c = append(c, k)
+		}
+	}
+
+	return f.filterIrrelevant(c)
+}
+
+func (f *fileChangeDetector) filterIrrelevant(in []string) []string {
+	var filtered []string
+	for _, v := range in {
+		if !f.irrelevantRe.MatchString(v) {
+			filtered = append(filtered, v)
+		}
+	}
+	return filtered
+}
+
+func (f *fileChangeDetector) PrepareNew() {
+	if f == nil {
+		return
+	}
+
+	f.Lock()
+	defer f.Unlock()
+
+	if f.current == nil {
+		f.current = make(map[string]string)
+		f.prev = make(map[string]string)
+		return
+	}
+
+	f.prev = make(map[string]string)
+	for k, v := range f.current {
+		f.prev[k] = v
+	}
+	f.current = make(map[string]string)
+}
+
+func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {
 
 	if c.DepsCfg == nil {
 		c.DepsCfg = &deps.DepsCfg{}
 	}
 
+	if c.logger != nil {
+		// Truncate the error log if this is a reload.
+		c.logger.Reset()
+	}
+
 	cfg := c.DepsCfg
 	c.configured = false
 	cfg.Running = running
@@ -142,6 +267,8 @@ func (c *commandeer) loadConfig(running bool) error {
 		sourceFs = c.DepsCfg.Fs.Source
 	}
 
+	environment := c.h.getEnvironment(running)
+
 	doWithConfig := func(cfg config.Provider) error {
 
 		if c.ftch != nil {
@@ -149,7 +276,7 @@ func (c *commandeer) loadConfig(running bool) error {
 		}
 
 		cfg.Set("workingDir", dir)
-
+		cfg.Set("environment", environment)
 		return nil
 	}
 
@@ -162,22 +289,44 @@ func (c *commandeer) loadConfig(running bool) error {
 		return err
 	}
 
+	configPath := c.h.source
+	if configPath == "" {
+		configPath = dir
+	}
 	config, configFiles, err := hugolib.LoadConfig(
-		hugolib.ConfigSourceDescriptor{Fs: sourceFs, Path: c.h.source, WorkingDir: dir, Filename: c.h.cfgFile},
+		hugolib.ConfigSourceDescriptor{
+			Fs:           sourceFs,
+			Path:         configPath,
+			WorkingDir:   dir,
+			Filename:     c.h.cfgFile,
+			AbsConfigDir: c.h.getConfigDir(dir),
+			Environ:      os.Environ(),
+			Environment:  environment},
 		doWithCommandeer,
 		doWithConfig)
 
 	if err != nil {
-		return err
+		if mustHaveConfigFile {
+			return err
+		}
+		if err != hugolib.ErrNoConfigFile && !modules.IsNotExist(err) {
+			return err
+		}
+
 	}
 
 	c.configFiles = configFiles
 
-	if l, ok := c.Cfg.Get("languagesSorted").(helpers.Languages); ok {
+	if l, ok := c.Cfg.Get("languagesSorted").(langs.Languages); ok {
 		c.languagesConfigured = true
 		c.languages = l
 	}
 
+	// Set some commonly used flags
+	c.doLiveReload = running && !c.Cfg.GetBool("disableLiveReload")
+	c.fastRenderMode = c.doLiveReload && !c.Cfg.GetBool("disableFastRender")
+	c.showErrorInBrowser = c.doLiveReload && !c.Cfg.GetBool("disableBrowserError")
+
 	// This is potentially double work, but we need to do this one more time now
 	// that all the languages have been configured.
 	if c.doWithCommandeer != nil {
@@ -186,12 +335,13 @@ func (c *commandeer) loadConfig(running bool) error {
 		}
 	}
 
-	logger, err := c.createLogger(config)
+	logger, err := c.createLogger(config, running)
 	if err != nil {
 		return err
 	}
 
 	cfg.Logger = logger
+	c.logger = logger
 
 	createMemFs := config.GetBool("renderToMemory")
 
@@ -203,49 +353,62 @@ func (c *commandeer) loadConfig(running bool) error {
 	c.fsCreate.Do(func() {
 		fs := hugofs.NewFrom(sourceFs, config)
 
-		// Hugo writes the output to memory instead of the disk.
-		if createMemFs {
+		if c.destinationFs != nil {
+			// Need to reuse the destination on server rebuilds.
+			fs.Destination = c.destinationFs
+		} else if createMemFs {
+			// Hugo writes the output to memory instead of the disk.
 			fs.Destination = new(afero.MemMapFs)
 		}
 
+		if c.fastRenderMode {
+			// For now, fast render mode only. It should, however, be fast enough
+			// for the full variant, too.
+			changeDetector := &fileChangeDetector{
+				// We use this detector to decide to do a Hot reload of a single path or not.
+				// We need to filter out source maps and possibly some other to be able
+				// to make that decision.
+				irrelevantRe: regexp.MustCompile(`\.map$`),
+			}
+
+			changeDetector.PrepareNew()
+			fs.Destination = hugofs.NewHashingFs(fs.Destination, changeDetector)
+			c.changeDetector = changeDetector
+		}
+
+		if c.Cfg.GetBool("logPathWarnings") {
+			fs.Destination = hugofs.NewCreateCountingFs(fs.Destination)
+		}
+
+		// To debug hard-to-find path issues.
+		//fs.Destination = hugofs.NewStacktracerFs(fs.Destination, `fr/fr`)
+
 		err = c.initFs(fs)
+		if err != nil {
+			close(c.created)
+			return
+		}
+
+		var h *hugolib.HugoSites
+
+		h, err = hugolib.NewHugoSites(*c.DepsCfg)
+		c.hugoSites = h
+		close(c.created)
+
 	})
 
 	if err != nil {
 		return err
 	}
 
-	cacheDir := config.GetString("cacheDir")
-	if cacheDir != "" {
-		if helpers.FilePathSeparator != cacheDir[len(cacheDir)-1:] {
-			cacheDir = cacheDir + helpers.FilePathSeparator
-		}
-		isDir, err := helpers.DirExists(cacheDir, sourceFs)
-		utils.CheckErr(cfg.Logger, err)
-		if !isDir {
-			mkdir(cacheDir)
-		}
-		config.Set("cacheDir", cacheDir)
-	} else {
-		config.Set("cacheDir", helpers.GetTempDir("hugo_cache", sourceFs))
+	cacheDir, err := helpers.GetCacheDir(sourceFs, config)
+	if err != nil {
+		return err
 	}
+	config.Set("cacheDir", cacheDir)
 
 	cfg.Logger.INFO.Println("Using config file:", config.ConfigFileUsed())
 
-	themeDir := c.PathSpec().GetThemeDir()
-	if themeDir != "" {
-		if _, err := sourceFs.Stat(themeDir); os.IsNotExist(err) {
-			return newSystemError("Unable to find theme Directory:", themeDir)
-		}
-	}
-
-	themeVersionMismatch, minVersion := c.isThemeVsHugoVersionMismatch(sourceFs)
-
-	if themeVersionMismatch {
-		cfg.Logger.ERROR.Printf("Current theme does not support Hugo version %s. Minimum version required is %s\n",
-			helpers.CurrentHugoVersion.ReleaseVersion(), minVersion)
-	}
-
 	return nil
 
 }

commands/commands.go

@@ -1,4 +1,4 @@
-// Copyright 2017 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,11 +14,15 @@
 package commands
 
 import (
+	"os"
+
+	"github.com/gohugoio/hugo/hugolib/paths"
+
+	"github.com/gohugoio/hugo/common/hugo"
+	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/spf13/cobra"
-
-	"github.com/spf13/nitro"
 )
 
 type commandsBuilder struct {
@@ -43,13 +47,14 @@ func (b *commandsBuilder) addAll() *commandsBuilder {
 		newEnvCmd(),
 		newConfigCmd(),
 		newCheckCmd(),
-		b.newBenchmarkCmd(),
+		newDeployCmd(),
 		newConvertCmd(),
-		newNewCmd(),
+		b.newNewCmd(),
 		newListCmd(),
 		newImportCmd(),
 		newGenCmd(),
 		createReleaser(),
+		b.newModCmd(),
 	)
 
 	return b
@@ -79,14 +84,14 @@ var _ commandsBuilderGetter = (*baseBuilderCmd)(nil)
 
 // Used in tests.
 type commandsBuilderGetter interface {
-	getCmmandsBuilder() *commandsBuilder
+	getCommandsBuilder() *commandsBuilder
 }
 type baseBuilderCmd struct {
 	*baseCmd
 	*commandsBuilder
 }
 
-func (b *baseBuilderCmd) getCmmandsBuilder() *commandsBuilder {
+func (b *baseBuilderCmd) getCommandsBuilder() *commandsBuilder {
 	return b.commandsBuilder
 }
 
@@ -148,7 +153,7 @@ Complete documentation is available at http://gohugo.io/.`,
 				return nil
 			}
 
-			c, err := initializeConfig(cc.buildWatch, &cc.hugoBuilderCommon, cc, cfgInit)
+			c, err := initializeConfig(true, cc.buildWatch, &cc.hugoBuilderCommon, cc, cfgInit)
 			if err != nil {
 				return err
 			}
@@ -159,11 +164,11 @@ Complete documentation is available at http://gohugo.io/.`,
 	})
 
 	cc.cmd.PersistentFlags().StringVar(&cc.cfgFile, "config", "", "config file (default is path/config.yaml|json|toml)")
+	cc.cmd.PersistentFlags().StringVar(&cc.cfgDir, "configDir", "config", "config dir")
 	cc.cmd.PersistentFlags().BoolVar(&cc.quiet, "quiet", false, "build in quiet mode")
 
 	// Set bash-completion
-	validConfigFilenames := []string{"json", "js", "yaml", "yml", "toml", "tml"}
-	_ = cc.cmd.PersistentFlags().SetAnnotation("config", cobra.BashCompFilenameExt, validConfigFilenames)
+	_ = cc.cmd.PersistentFlags().SetAnnotation("config", cobra.BashCompFilenameExt, config.ValidConfigFileExtensions)
 
 	cc.cmd.PersistentFlags().BoolVarP(&cc.verbose, "verbose", "v", false, "verbose output")
 	cc.cmd.PersistentFlags().BoolVarP(&cc.debug, "debug", "", false, "debug output")
@@ -185,13 +190,20 @@ Complete documentation is available at http://gohugo.io/.`,
 }
 
 type hugoBuilderCommon struct {
-	source  string
-	baseURL string
+	source      string
+	baseURL     string
+	environment string
 
 	buildWatch bool
 
 	gc bool
 
+	// Profile flags (for debugging of performance problems)
+	cpuprofile   string
+	memprofile   string
+	mutexprofile string
+	traceprofile string
+
 	// TODO(bep) var vs string
 	logging    bool
 	verbose    bool
@@ -200,40 +212,83 @@ type hugoBuilderCommon struct {
 	quiet      bool
 
 	cfgFile string
+	cfgDir  string
 	logFile string
 }
 
+func (cc *hugoBuilderCommon) getConfigDir(baseDir string) string {
+	if cc.cfgDir != "" {
+		return paths.AbsPathify(baseDir, cc.cfgDir)
+	}
+
+	if v, found := os.LookupEnv("HUGO_CONFIGDIR"); found {
+		return paths.AbsPathify(baseDir, v)
+	}
+
+	return paths.AbsPathify(baseDir, "config")
+}
+
+func (cc *hugoBuilderCommon) getEnvironment(isServer bool) string {
+	if cc.environment != "" {
+		return cc.environment
+	}
+
+	if v, found := os.LookupEnv("HUGO_ENVIRONMENT"); found {
+		return v
+	}
+
+	if isServer {
+		return hugo.EnvironmentDevelopment
+	}
+
+	return hugo.EnvironmentProduction
+}
+
+func (cc *hugoBuilderCommon) handleCommonBuilderFlags(cmd *cobra.Command) {
+	cmd.PersistentFlags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")
+	cmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{})
+	cmd.PersistentFlags().StringVarP(&cc.environment, "environment", "e", "", "build environment")
+	cmd.PersistentFlags().StringP("themesDir", "", "", "filesystem path to themes directory")
+	cmd.PersistentFlags().BoolP("ignoreVendor", "", false, "ignores any _vendor directory")
+}
+
 func (cc *hugoBuilderCommon) handleFlags(cmd *cobra.Command) {
+	cc.handleCommonBuilderFlags(cmd)
 	cmd.Flags().Bool("cleanDestinationDir", false, "remove files from destination not found in static directories")
 	cmd.Flags().BoolP("buildDrafts", "D", false, "include content marked as draft")
 	cmd.Flags().BoolP("buildFuture", "F", false, "include content with publishdate in the future")
 	cmd.Flags().BoolP("buildExpired", "E", false, "include expired content")
-	cmd.Flags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")
 	cmd.Flags().StringP("contentDir", "c", "", "filesystem path to content directory")
 	cmd.Flags().StringP("layoutDir", "l", "", "filesystem path to layout directory")
 	cmd.Flags().StringP("cacheDir", "", "", "filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/")
 	cmd.Flags().BoolP("ignoreCache", "", false, "ignores the cache directory")
 	cmd.Flags().StringP("destination", "d", "", "filesystem path to write files to")
-	cmd.Flags().StringP("theme", "t", "", "theme to use (located in /themes/THEMENAME/)")
-	cmd.Flags().StringP("themesDir", "", "", "filesystem path to themes directory")
-	cmd.Flags().Bool("uglyURLs", false, "(deprecated) if true, use /filename.html instead of /filename/")
-	cmd.Flags().Bool("canonifyURLs", false, "(deprecated) if true, all relative URLs will be canonicalized using baseURL")
+	cmd.Flags().StringSliceP("theme", "t", []string{}, "themes to use (located in /themes/THEMENAME/)")
 	cmd.Flags().StringVarP(&cc.baseURL, "baseURL", "b", "", "hostname (and path) to the root, e.g. http://spf13.com/")
 	cmd.Flags().Bool("enableGitInfo", false, "add Git revision, date and author info to the pages")
 	cmd.Flags().BoolVar(&cc.gc, "gc", false, "enable to run some cleanup tasks (remove unused cache files) after the build")
 
-	cmd.Flags().BoolVar(&nitro.AnalysisOn, "stepAnalysis", false, "display memory and timing of different steps of the program")
 	cmd.Flags().Bool("templateMetrics", false, "display metrics about template executions")
 	cmd.Flags().Bool("templateMetricsHints", false, "calculate some improvement hints when combined with --templateMetrics")
-	cmd.Flags().Bool("pluralizeListTitles", true, "(deprecated) pluralize titles in lists using inflect")
-	cmd.Flags().Bool("preserveTaxonomyNames", false, `(deprecated) preserve taxonomy names as written ("Gérard Depardieu" vs "gerard-depardieu")`)
 	cmd.Flags().BoolP("forceSyncStatic", "", false, "copy all files when static is changed.")
 	cmd.Flags().BoolP("noTimes", "", false, "don't sync modification time of files")
 	cmd.Flags().BoolP("noChmod", "", false, "don't sync permission mode of files")
 	cmd.Flags().BoolP("i18n-warnings", "", false, "print missing translations")
+	cmd.Flags().BoolP("path-warnings", "", false, "print warnings on duplicate target paths etc.")
+	cmd.Flags().StringVarP(&cc.cpuprofile, "profile-cpu", "", "", "write cpu profile to `file`")
+	cmd.Flags().StringVarP(&cc.memprofile, "profile-mem", "", "", "write memory profile to `file`")
+	cmd.Flags().StringVarP(&cc.mutexprofile, "profile-mutex", "", "", "write Mutex profile to `file`")
+	cmd.Flags().StringVarP(&cc.traceprofile, "trace", "", "", "write trace to `file` (not useful in general)")
+
+	// Hide these for now.
+	cmd.Flags().MarkHidden("profile-cpu")
+	cmd.Flags().MarkHidden("profile-mem")
+	cmd.Flags().MarkHidden("profile-mutex")
 
 	cmd.Flags().StringSlice("disableKinds", []string{}, "disable different kind of pages (home, RSS etc.)")
 
+	cmd.Flags().Bool("minify", false, "minify any supported output format (HTML, XML etc.)")
+
 	// Set bash-completion.
 	// Each flag must first be defined before using the SetAnnotation() call.
 	_ = cmd.Flags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{})
@@ -241,3 +296,17 @@ func (cc *hugoBuilderCommon) handleFlags(cmd *cobra.Command) {
 	_ = cmd.Flags().SetAnnotation("destination", cobra.BashCompSubdirsInDir, []string{})
 	_ = cmd.Flags().SetAnnotation("theme", cobra.BashCompSubdirsInDir, []string{"themes"})
 }
+
+func checkErr(logger *loggers.Logger, err error, s ...string) {
+	if err == nil {
+		return
+	}
+	if len(s) == 0 {
+		logger.CRITICAL.Println(err)
+		return
+	}
+	for _, message := range s {
+		logger.ERROR.Println(message)
+	}
+	logger.ERROR.Println(err)
+}

commands/commands_test.go

@@ -1,4 +1,4 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,32 +20,34 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/gohugoio/hugo/common/types"
+
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/stretchr/testify/require"
+	qt "github.com/frankban/quicktest"
 )
 
 func TestExecute(t *testing.T) {
 
-	assert := require.New(t)
+	c := qt.New(t)
 
-	dir, err := createSimpleTestSite(t)
-	assert.NoError(err)
+	dir, err := createSimpleTestSite(t, testSiteConfig{})
+	c.Assert(err, qt.IsNil)
 
 	defer func() {
 		os.RemoveAll(dir)
 	}()
 
 	resp := Execute([]string{"-s=" + dir})
-	assert.NoError(resp.Err)
+	c.Assert(resp.Err, qt.IsNil)
 	result := resp.Result
-	assert.True(len(result.Sites) == 1)
-	assert.True(len(result.Sites[0].RegularPages) == 1)
+	c.Assert(len(result.Sites) == 1, qt.Equals, true)
+	c.Assert(len(result.Sites[0].RegularPages()) == 1, qt.Equals, true)
 }
 
 func TestCommandsPersistentFlags(t *testing.T) {
-	assert := require.New(t)
+	c := qt.New(t)
 
 	noOpRunE := func(cmd *cobra.Command, args []string) error {
 		return nil
@@ -56,8 +58,11 @@ func TestCommandsPersistentFlags(t *testing.T) {
 		check func(command []cmder)
 	}{{[]string{"server",
 		"--config=myconfig.toml",
+		"--configDir=myconfigdir",
 		"--contentDir=mycontent",
 		"--disableKinds=page,home",
+		"--environment=testing",
+		"--configDir=myconfigdir",
 		"--layoutDir=mylayouts",
 		"--theme=mytheme",
 		"--gc",
@@ -72,14 +77,16 @@ func TestCommandsPersistentFlags(t *testing.T) {
 		"--port=1366",
 		"--renderToDisk",
 		"--source=mysource",
-		"--uglyURLs"}, func(commands []cmder) {
+		"--path-warnings",
+	}, func(commands []cmder) {
 		var sc *serverCmd
 		for _, command := range commands {
 			if b, ok := command.(commandsBuilderGetter); ok {
-				v := b.getCmmandsBuilder().hugoBuilderCommon
-				assert.Equal("myconfig.toml", v.cfgFile)
-				assert.Equal("mysource", v.source)
-				assert.Equal("https://example.com/b/", v.baseURL)
+				v := b.getCommandsBuilder().hugoBuilderCommon
+				c.Assert(v.cfgFile, qt.Equals, "myconfig.toml")
+				c.Assert(v.cfgDir, qt.Equals, "myconfigdir")
+				c.Assert(v.source, qt.Equals, "mysource")
+				c.Assert(v.baseURL, qt.Equals, "https://example.com/b/")
 			}
 
 			if srvCmd, ok := command.(*serverCmd); ok {
@@ -87,29 +94,32 @@ func TestCommandsPersistentFlags(t *testing.T) {
 			}
 		}
 
-		assert.NotNil(sc)
-		assert.True(sc.navigateToChanged)
-		assert.True(sc.disableLiveReload)
-		assert.True(sc.noHTTPCache)
-		assert.True(sc.renderToDisk)
-		assert.Equal(1366, sc.serverPort)
+		c.Assert(sc, qt.Not(qt.IsNil))
+		c.Assert(sc.navigateToChanged, qt.Equals, true)
+		c.Assert(sc.disableLiveReload, qt.Equals, true)
+		c.Assert(sc.noHTTPCache, qt.Equals, true)
+		c.Assert(sc.renderToDisk, qt.Equals, true)
+		c.Assert(sc.serverPort, qt.Equals, 1366)
+		c.Assert(sc.environment, qt.Equals, "testing")
 
 		cfg := viper.New()
 		sc.flagsToConfig(cfg)
-		assert.Equal("/tmp/mydestination", cfg.GetString("publishDir"))
-		assert.Equal("mycontent", cfg.GetString("contentDir"))
-		assert.Equal("mylayouts", cfg.GetString("layoutDir"))
-		assert.Equal("mytheme", cfg.GetString("theme"))
-		assert.Equal("mythemes", cfg.GetString("themesDir"))
-		assert.Equal("https://example.com/b/", cfg.GetString("baseURL"))
+		c.Assert(cfg.GetString("publishDir"), qt.Equals, "/tmp/mydestination")
+		c.Assert(cfg.GetString("contentDir"), qt.Equals, "mycontent")
+		c.Assert(cfg.GetString("layoutDir"), qt.Equals, "mylayouts")
+		c.Assert(cfg.GetStringSlice("theme"), qt.DeepEquals, []string{"mytheme"})
+		c.Assert(cfg.GetString("themesDir"), qt.Equals, "mythemes")
+		c.Assert(cfg.GetString("baseURL"), qt.Equals, "https://example.com/b/")
 
-		assert.Equal([]string{"page", "home"}, cfg.Get("disableKinds"))
+		c.Assert(cfg.Get("disableKinds"), qt.DeepEquals, []string{"page", "home"})
 
-		assert.True(cfg.GetBool("uglyURLs"))
-		assert.True(cfg.GetBool("gc"))
+		c.Assert(cfg.GetBool("gc"), qt.Equals, true)
+
+		// The flag is named path-warnings
+		c.Assert(cfg.GetBool("logPathWarnings"), qt.Equals, true)
 
 		// The flag is named i18n-warnings
-		assert.True(cfg.GetBool("logI18nWarnings"))
+		c.Assert(cfg.GetBool("logI18nWarnings"), qt.Equals, true)
 
 	}}}
 
@@ -126,7 +136,7 @@ func TestCommandsPersistentFlags(t *testing.T) {
 		}
 		rootCmd := root.getCommand()
 		rootCmd.SetArgs(test.args)
-		assert.NoError(rootCmd.Execute())
+		c.Assert(rootCmd.Execute(), qt.IsNil)
 		test.check(b.commands)
 	}
 
@@ -134,13 +144,13 @@ func TestCommandsPersistentFlags(t *testing.T) {
 
 func TestCommandsExecute(t *testing.T) {
 
-	assert := require.New(t)
+	c := qt.New(t)
 
-	dir, err := createSimpleTestSite(t)
-	assert.NoError(err)
+	dir, err := createSimpleTestSite(t, testSiteConfig{})
+	c.Assert(err, qt.IsNil)
 
 	dirOut, err := ioutil.TempDir("", "hugo-cli-out")
-	assert.NoError(err)
+	c.Assert(err, qt.IsNil)
 
 	defer func() {
 		os.RemoveAll(dir)
@@ -161,7 +171,6 @@ func TestCommandsExecute(t *testing.T) {
 		{nil, []string{sourceFlag}, ""},
 		{nil, []string{sourceFlag, "--renderToMemory"}, ""},
 		{[]string{"config"}, []string{sourceFlag}, ""},
-		{[]string{"benchmark"}, []string{sourceFlag, "-n=1"}, ""},
 		{[]string{"convert", "toTOML"}, []string{sourceFlag, "-o=" + filepath.Join(dirOut, "toml")}, ""},
 		{[]string{"convert", "toYAML"}, []string{sourceFlag, "-o=" + filepath.Join(dirOut, "yaml")}, ""},
 		{[]string{"convert", "toJSON"}, []string{sourceFlag, "-o=" + filepath.Join(dirOut, "json")}, ""},
@@ -180,8 +189,8 @@ func TestCommandsExecute(t *testing.T) {
 	}
 
 	for _, test := range tests {
-
-		hugoCmd := newCommandsBuilder().addAll().build().getCommand()
+		b := newCommandsBuilder().addAll().build()
+		hugoCmd := b.getCommand()
 		test.flags = append(test.flags, "--quiet")
 		hugoCmd.SetArgs(append(test.commands, test.flags...))
 
@@ -191,31 +200,54 @@ func TestCommandsExecute(t *testing.T) {
 
 		_, err := hugoCmd.ExecuteC()
 		if test.expectErrToContain != "" {
-			assert.Error(err, fmt.Sprintf("%v", test.commands))
-			assert.Contains(err.Error(), test.expectErrToContain)
+			c.Assert(err, qt.Not(qt.IsNil))
+			c.Assert(err.Error(), qt.Contains, test.expectErrToContain)
 		} else {
-			assert.NoError(err, fmt.Sprintf("%v", test.commands))
+			c.Assert(err, qt.IsNil)
+		}
+
+		// Assert that we have not left any development debug artifacts in
+		// the code.
+		if b.c != nil {
+			_, ok := b.c.destinationFs.(types.DevMarker)
+			c.Assert(ok, qt.Equals, false)
 		}
 
 	}
 
 }
 
-func createSimpleTestSite(t *testing.T) (string, error) {
+type testSiteConfig struct {
+	configTOML string
+	contentDir string
+}
+
+func createSimpleTestSite(t *testing.T, cfg testSiteConfig) (string, error) {
 	d, e := ioutil.TempDir("", "hugo-cli")
 	if e != nil {
 		return "", e
 	}
 
-	// Just the basic. These are for CLI tests, not site testing.
-	writeFile(t, filepath.Join(d, "config.toml"), `
+	cfgStr := `
 
 baseURL = "https://example.org"
 title = "Hugo Commands"
 
-`)
+`
 
-	writeFile(t, filepath.Join(d, "content", "p1.md"), `
+	contentDir := "content"
+
+	if cfg.configTOML != "" {
+		cfgStr = cfg.configTOML
+	}
+	if cfg.contentDir != "" {
+		contentDir = cfg.contentDir
+	}
+
+	// Just the basic. These are for CLI tests, not site testing.
+	writeFile(t, filepath.Join(d, "config.toml"), cfgStr)
+
+	writeFile(t, filepath.Join(d, contentDir, "p1.md"), `
 ---
 title: "P1"
 weight: 1
@@ -234,6 +266,7 @@ Single: {{ .Title }}
 	writeFile(t, filepath.Join(d, "layouts", "_default", "list.html"), `
 
 List: {{ .Title }}
+Environment: {{ hugo.Environment }}
 
 `)
 

commands/config.go

@@ -14,8 +14,17 @@
 package commands
 
 import (
+	"encoding/json"
+	"os"
 	"reflect"
+	"regexp"
 	"sort"
+	"strings"
+
+	"github.com/gohugoio/hugo/parser"
+	"github.com/gohugoio/hugo/parser/metadecoders"
+
+	"github.com/gohugoio/hugo/modules"
 
 	"github.com/spf13/cobra"
 	jww "github.com/spf13/jwalterweatherman"
@@ -38,29 +47,58 @@ func newConfigCmd() *configCmd {
 		RunE:  cc.printConfig,
 	})
 
-	cc.cmd.Flags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")
+	cc.cmd.PersistentFlags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")
+
+	printMountsCmd := &cobra.Command{
+		Use:   "mounts",
+		Short: "Print the configured file mounts",
+		RunE:  cc.printMounts,
+	}
+
+	cc.cmd.AddCommand(printMountsCmd)
 
 	return cc
 }
 
-func (c *configCmd) printConfig(cmd *cobra.Command, args []string) error {
-	cfg, err := initializeConfig(false, &c.hugoBuilderCommon, c, nil)
+func (c *configCmd) printMounts(cmd *cobra.Command, args []string) error {
+	cfg, err := initializeConfig(true, false, &c.hugoBuilderCommon, c, nil)
+	if err != nil {
+		return err
+	}
 
+	allModules := cfg.Cfg.Get("allmodules").(modules.Modules)
+
+	for _, m := range allModules {
+		if err := parser.InterfaceToConfig(&modMounts{m: m}, metadecoders.JSON, os.Stdout); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *configCmd) printConfig(cmd *cobra.Command, args []string) error {
+	cfg, err := initializeConfig(true, false, &c.hugoBuilderCommon, c, nil)
 	if err != nil {
 		return err
 	}
 
 	allSettings := cfg.Cfg.(*viper.Viper).AllSettings()
 
-	var separator string
-	if allSettings["metadataformat"] == "toml" {
+	// We need to clean up this, but we store objects in the config that
+	// isn't really interesting to the end user, so filter these.
+	ignoreKeysRe := regexp.MustCompile("client|sorted|filecacheconfigs|allmodules|multilingual")
+
+	separator := ": "
+
+	if len(cfg.configFiles) > 0 && strings.HasSuffix(cfg.configFiles[0], ".toml") {
 		separator = " = "
-	} else {
-		separator = ": "
 	}
 
 	var keys []string
 	for k := range allSettings {
+		if ignoreKeysRe.MatchString(k) {
+			continue
+		}
 		keys = append(keys, k)
 	}
 	sort.Strings(keys)
@@ -75,3 +113,35 @@ func (c *configCmd) printConfig(cmd *cobra.Command, args []string) error {
 
 	return nil
 }
+
+type modMounts struct {
+	m modules.Module
+}
+
+type modMount struct {
+	Source string `json:"source"`
+	Target string `json:"target"`
+	Lang   string `json:"lang,omitempty"`
+}
+
+func (m *modMounts) MarshalJSON() ([]byte, error) {
+	var mounts []modMount
+
+	for _, mount := range m.m.Mounts() {
+		mounts = append(mounts, modMount{
+			Source: mount.Source,
+			Target: mount.Target,
+			Lang:   mount.Lang,
+		})
+	}
+
+	return json.Marshal(&struct {
+		Path   string     `json:"path"`
+		Dir    string     `json:"dir"`
+		Mounts []modMount `json:"mounts"`
+	}{
+		Path:   m.m.Path(),
+		Dir:    m.m.Dir(),
+		Mounts: mounts,
+	})
+}

commands/convert.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,17 +14,28 @@
 package commands
 
 import (
+	"bytes"
 	"fmt"
+	"io"
+	"strings"
 	"time"
 
-	src "github.com/gohugoio/hugo/source"
+	"github.com/gohugoio/hugo/resources/page"
+
+	"github.com/gohugoio/hugo/hugofs"
+
+	"github.com/gohugoio/hugo/helpers"
+
+	"github.com/gohugoio/hugo/parser"
+	"github.com/gohugoio/hugo/parser/metadecoders"
+	"github.com/gohugoio/hugo/parser/pageparser"
+
+	"github.com/pkg/errors"
 
 	"github.com/gohugoio/hugo/hugolib"
 
 	"path/filepath"
 
-	"github.com/gohugoio/hugo/parser"
-	"github.com/spf13/cast"
 	"github.com/spf13/cobra"
 )
 
@@ -60,7 +71,7 @@ See convert's subcommands toJSON, toTOML and toYAML for more information.`,
 			Long: `toJSON converts all front matter in the content directory
 to use JSON for the front matter.`,
 			RunE: func(cmd *cobra.Command, args []string) error {
-				return cc.convertContents(rune([]byte(parser.JSONLead)[0]))
+				return cc.convertContents(metadecoders.JSON)
 			},
 		},
 		&cobra.Command{
@@ -69,7 +80,7 @@ to use JSON for the front matter.`,
 			Long: `toTOML converts all front matter in the content directory
 to use TOML for the front matter.`,
 			RunE: func(cmd *cobra.Command, args []string) error {
-				return cc.convertContents(rune([]byte(parser.TOMLLead)[0]))
+				return cc.convertContents(metadecoders.TOML)
 			},
 		},
 		&cobra.Command{
@@ -78,7 +89,7 @@ to use TOML for the front matter.`,
 			Long: `toYAML converts all front matter in the content directory
 to use YAML for the front matter.`,
 			RunE: func(cmd *cobra.Command, args []string) error {
-				return cc.convertContents(rune([]byte(parser.YAMLLead)[0]))
+				return cc.convertContents(metadecoders.YAML)
 			},
 		},
 	)
@@ -91,16 +102,18 @@ to use YAML for the front matter.`,
 	return cc
 }
 
-func (cc *convertCmd) convertContents(mark rune) error {
+func (cc *convertCmd) convertContents(format metadecoders.Format) error {
 	if cc.outputDir == "" && !cc.unsafe {
 		return newUserError("Unsafe operation not allowed, use --unsafe or set a different output path")
 	}
 
-	c, err := initializeConfig(false, &cc.hugoBuilderCommon, cc, nil)
+	c, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, nil)
 	if err != nil {
 		return err
 	}
 
+	c.Cfg.Set("buildDrafts", true)
+
 	h, err := hugolib.NewHugoSites(*c.DepsCfg)
 	if err != nil {
 		return err
@@ -112,83 +125,126 @@ func (cc *convertCmd) convertContents(mark rune) error {
 
 	site := h.Sites[0]
 
-	site.Log.FEEDBACK.Println("processing", len(site.AllPages), "content files")
-	for _, p := range site.AllPages {
-		if err := cc.convertAndSavePage(p, site, mark); err != nil {
+	site.Log.FEEDBACK.Println("processing", len(site.AllPages()), "content files")
+	for _, p := range site.AllPages() {
+		if err := cc.convertAndSavePage(p, site, format); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, mark rune) error {
+func (cc *convertCmd) convertAndSavePage(p page.Page, site *hugolib.Site, targetFormat metadecoders.Format) error {
 	// The resources are not in .Site.AllPages.
-	for _, r := range p.Resources.ByType("page") {
-		if err := cc.convertAndSavePage(r.(*hugolib.Page), site, mark); err != nil {
+	for _, r := range p.Resources().ByType("page") {
+		if err := cc.convertAndSavePage(r.(page.Page), site, targetFormat); err != nil {
 			return err
 		}
 	}
 
-	if p.Filename() == "" {
+	if p.File().IsZero() {
 		// No content file.
 		return nil
 	}
 
-	site.Log.INFO.Println("Attempting to convert", p.LogicalName())
-	newPage, err := site.NewPage(p.LogicalName())
-	if err != nil {
-		return err
-	}
+	errMsg := fmt.Errorf("Error processing file %q", p.Path())
 
-	f, _ := p.File.(src.ReadableFile)
-	file, err := f.Open()
+	site.Log.INFO.Println("Attempting to convert", p.File().Filename())
+
+	f := p.File()
+	file, err := f.FileInfo().Meta().Open()
 	if err != nil {
-		site.Log.ERROR.Println("Error reading file:", p.Path())
+		site.Log.ERROR.Println(errMsg)
 		file.Close()
 		return nil
 	}
 
-	psr, err := parser.ReadFrom(file)
+	pf, err := parseContentFile(file)
 	if err != nil {
-		site.Log.ERROR.Println("Error processing file:", p.Path())
+		site.Log.ERROR.Println(errMsg)
 		file.Close()
 		return err
 	}
 
 	file.Close()
 
-	metadata, err := psr.Metadata()
+	// better handling of dates in formats that don't have support for them
+	if pf.frontMatterFormat == metadecoders.JSON || pf.frontMatterFormat == metadecoders.YAML || pf.frontMatterFormat == metadecoders.TOML {
+		for k, v := range pf.frontMatter {
+			switch vv := v.(type) {
+			case time.Time:
+				pf.frontMatter[k] = vv.Format(time.RFC3339)
+			}
+		}
+	}
+
+	var newContent bytes.Buffer
+	err = parser.InterfaceToFrontMatter(pf.frontMatter, targetFormat, &newContent)
 	if err != nil {
-		site.Log.ERROR.Println("Error processing file:", p.Path())
+		site.Log.ERROR.Println(errMsg)
 		return err
 	}
 
-	// better handling of dates in formats that don't have support for them
-	if mark == parser.FormatToLeadRune("json") || mark == parser.FormatToLeadRune("yaml") || mark == parser.FormatToLeadRune("toml") {
-		newMetadata := cast.ToStringMap(metadata)
-		for k, v := range newMetadata {
-			switch vv := v.(type) {
-			case time.Time:
-				newMetadata[k] = vv.Format(time.RFC3339)
-			}
-		}
-		metadata = newMetadata
-	}
+	newContent.Write(pf.content)
 
-	newPage.SetSourceContent(psr.Content())
-	if err = newPage.SetSourceMetaData(metadata, mark); err != nil {
-		site.Log.ERROR.Printf("Failed to set source metadata for file %q: %s. For more info see For more info see https://github.com/gohugoio/hugo/issues/2458", newPage.FullFilePath(), err)
-		return nil
-	}
+	newFilename := p.File().Filename()
 
-	newFilename := p.Filename()
 	if cc.outputDir != "" {
-		newFilename = filepath.Join(cc.outputDir, p.Dir(), newPage.LogicalName())
+		contentDir := strings.TrimSuffix(newFilename, p.Path())
+		contentDir = filepath.Base(contentDir)
+
+		newFilename = filepath.Join(cc.outputDir, contentDir, p.Path())
 	}
 
-	if err = newPage.SaveSourceAs(newFilename); err != nil {
-		return fmt.Errorf("Failed to save file %q: %s", newFilename, err)
+	fs := hugofs.Os
+	if err := helpers.WriteToDisk(newFilename, &newContent, fs); err != nil {
+		return errors.Wrapf(err, "Failed to save file %q:", newFilename)
 	}
 
 	return nil
 }
+
+type parsedFile struct {
+	frontMatterFormat metadecoders.Format
+	frontMatterSource []byte
+	frontMatter       map[string]interface{}
+
+	// Everything after Front Matter
+	content []byte
+}
+
+func parseContentFile(r io.Reader) (parsedFile, error) {
+	var pf parsedFile
+
+	psr, err := pageparser.Parse(r, pageparser.Config{})
+	if err != nil {
+		return pf, err
+	}
+
+	iter := psr.Iterator()
+
+	walkFn := func(item pageparser.Item) bool {
+		if pf.frontMatterSource != nil {
+			// The rest is content.
+			pf.content = psr.Input()[item.Pos:]
+			// Done
+			return false
+		} else if item.IsFrontMatter() {
+			pf.frontMatterFormat = metadecoders.FormatFromFrontMatterType(item.Type)
+			pf.frontMatterSource = item.Val
+		}
+		return true
+
+	}
+
+	iter.PeekWalk(walkFn)
+
+	metadata, err := metadecoders.Default.UnmarshalToMap(pf.frontMatterSource, pf.frontMatterFormat)
+	if err != nil {
+		return pf, err
+	}
+	pf.frontMatter = metadata
+
+	return pf, nil
+
+}

commands/deploy.go

@@ -0,0 +1,77 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package commands
+
+import (
+	"context"
+
+	"github.com/gohugoio/hugo/deploy"
+	"github.com/spf13/cobra"
+)
+
+var _ cmder = (*deployCmd)(nil)
+
+// deployCmd supports deploying sites to Cloud providers.
+type deployCmd struct {
+	hugoBuilderCommon
+	*baseCmd
+}
+
+// TODO: In addition to the "deploy" command, consider adding a "--deploy"
+// flag for the default command; this would build the site and then deploy it.
+// It's not obvious how to do this; would all of the deploy-specific flags
+// have to exist at the top level as well?
+
+// TODO:  The output files change every time "hugo" is executed, it looks
+// like because of map order randomization. This means that you can
+// run "hugo && hugo deploy" again and again and upload new stuff every time. Is
+// this intended?
+
+func newDeployCmd() *deployCmd {
+	cc := &deployCmd{}
+
+	cc.baseCmd = newBaseCmd(&cobra.Command{
+		Use:   "deploy",
+		Short: "Deploy your site to a Cloud provider.",
+		Long: `Deploy your site to a Cloud provider.
+
+See https://gohugo.io/hosting-and-deployment/hugo-deploy/ for detailed
+documentation.
+`,
+
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfgInit := func(c *commandeer) error {
+				return nil
+			}
+			comm, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, cfgInit)
+			if err != nil {
+				return err
+			}
+			deployer, err := deploy.New(comm.Cfg, comm.hugo().PathSpec.PublishFs)
+			if err != nil {
+				return err
+			}
+			return deployer.Deploy(context.Background())
+		},
+	})
+
+	cc.cmd.Flags().String("target", "", "target deployment from deployments section in config file; defaults to the first one")
+	cc.cmd.Flags().Bool("confirm", false, "ask for confirmation before making changes to the target")
+	cc.cmd.Flags().Bool("dryRun", false, "dry run")
+	cc.cmd.Flags().Bool("force", false, "force upload of all files")
+	cc.cmd.Flags().Bool("invalidateCDN", true, "invalidate the CDN cache via the cloudFrontDistributionID listed in the deployment target")
+	cc.cmd.Flags().Int("maxDeletes", 256, "maximum # of files to delete, or -1 to disable")
+
+	return cc
+}

commands/genman.go

@@ -17,6 +17,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/gohugoio/hugo/common/hugo"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/spf13/cobra"
@@ -45,7 +46,7 @@ in the "man" directory under the current directory.`,
 			header := &doc.GenManHeader{
 				Section: "1",
 				Manual:  "Hugo Manual",
-				Source:  fmt.Sprintf("Hugo %s", helpers.CurrentHugoVersion),
+				Source:  fmt.Sprintf("Hugo %s", hugo.CurrentVersion),
 			}
 			if !strings.HasSuffix(cc.genmandir, helpers.FilePathSeparator) {
 				cc.genmandir += helpers.FilePathSeparator

commands/hugo_test.go

@@ -0,0 +1,52 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package commands
+
+import (
+	"os"
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+// Issue #5662
+func TestHugoWithContentDirOverride(t *testing.T) {
+	c := qt.New(t)
+
+	hugoCmd := newCommandsBuilder().addAll().build()
+	cmd := hugoCmd.getCommand()
+
+	contentDir := "contentOverride"
+
+	cfgStr := `
+
+baseURL = "https://example.org"
+title = "Hugo Commands"
+
+contentDir = "thisdoesnotexist"
+
+`
+	dir, err := createSimpleTestSite(t, testSiteConfig{configTOML: cfgStr, contentDir: contentDir})
+	c.Assert(err, qt.IsNil)
+
+	defer func() {
+		os.RemoveAll(dir)
+	}()
+
+	cmd.SetArgs([]string{"-s=" + dir, "-c=" + contentDir})
+
+	_, err = cmd.ExecuteC()
+	c.Assert(err, qt.IsNil)
+
+}

commands/hugo_windows.go

@@ -23,5 +23,5 @@ func init() {
 
   You need to open cmd.exe and run Hugo from there.
   
-  Visit http://gohugo.io/ for more information.`
+  Visit https://gohugo.io/ for more information.`
 }

commands/import_jekyll.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@ package commands
 import (
 	"bytes"
 	"errors"
-	"io"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -24,6 +24,11 @@ import (
 	"strconv"
 	"strings"
 	"time"
+	"unicode"
+
+	"github.com/gohugoio/hugo/common/hugio"
+
+	"github.com/gohugoio/hugo/parser/metadecoders"
 
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
@@ -73,23 +78,23 @@ Import from Jekyll requires two paths, e.g. ` + "`hugo import jekyll jekyll_root
 func (i *importCmd) importFromJekyll(cmd *cobra.Command, args []string) error {
 
 	if len(args) < 2 {
-		return newUserError(`Import from Jekyll requires two paths, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.")
+		return newUserError(`import from jekyll requires two paths, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.")
 	}
 
 	jekyllRoot, err := filepath.Abs(filepath.Clean(args[0]))
 	if err != nil {
-		return newUserError("Path error:", args[0])
+		return newUserError("path error:", args[0])
 	}
 
 	targetDir, err := filepath.Abs(filepath.Clean(args[1]))
 	if err != nil {
-		return newUserError("Path error:", args[1])
+		return newUserError("path error:", args[1])
 	}
 
 	jww.INFO.Println("Import Jekyll from:", jekyllRoot, "to:", targetDir)
 
 	if strings.HasPrefix(filepath.Dir(targetDir), jekyllRoot) {
-		return newUserError("Target path should not be inside the Jekyll root, aborting.")
+		return newUserError("abort: target path should not be inside the Jekyll root")
 	}
 
 	forceImport, _ := cmd.Flags().GetBool("force")
@@ -97,7 +102,7 @@ func (i *importCmd) importFromJekyll(cmd *cobra.Command, args []string) error {
 	fs := afero.NewOsFs()
 	jekyllPostDirs, hasAnyPost := i.getJekyllDirInfo(fs, jekyllRoot)
 	if !hasAnyPost {
-		return errors.New("Your Jekyll root contains neither posts nor drafts, aborting.")
+		return errors.New("abort: jekyll root contains neither posts nor drafts")
 	}
 
 	site, err := i.createSiteFromJekyll(jekyllRoot, targetDir, jekyllPostDirs, forceImport)
@@ -109,7 +114,7 @@ func (i *importCmd) importFromJekyll(cmd *cobra.Command, args []string) error {
 	jww.FEEDBACK.Println("Importing...")
 
 	fileCount := 0
-	callback := func(path string, fi os.FileInfo, err error) error {
+	callback := func(path string, fi hugofs.FileMetaInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -120,7 +125,7 @@ func (i *importCmd) importFromJekyll(cmd *cobra.Command, args []string) error {
 
 		relPath, err := filepath.Rel(jekyllRoot, path)
 		if err != nil {
-			return newUserError("Get rel path error:", path)
+			return newUserError("get rel path error:", path)
 		}
 
 		relPath = filepath.ToSlash(relPath)
@@ -204,13 +209,13 @@ func (i *importCmd) createSiteFromJekyll(jekyllRoot, targetDir string, jekyllPos
 	fs := s.Fs.Source
 	if exists, _ := helpers.Exists(targetDir, fs); exists {
 		if isDir, _ := helpers.IsDir(targetDir, fs); !isDir {
-			return nil, errors.New("Target path \"" + targetDir + "\" already exists but not a directory")
+			return nil, errors.New("target path \"" + targetDir + "\" exists but is not a directory")
 		}
 
 		isEmpty, _ := helpers.IsEmpty(targetDir, fs)
 
 		if !isEmpty && !force {
-			return nil, errors.New("Target path \"" + targetDir + "\" already exists and is not empty")
+			return nil, errors.New("target path \"" + targetDir + "\" exists and is not empty")
 		}
 	}
 
@@ -253,7 +258,7 @@ func (i *importCmd) loadJekyllConfig(fs afero.Fs, jekyllRoot string) map[string]
 		return nil
 	}
 
-	c, err := parser.HandleYAMLMetaData(b)
+	c, err := metadecoders.Default.UnmarshalToMap(b, metadecoders.YAML)
 
 	if err != nil {
 		return nil
@@ -262,7 +267,7 @@ func (i *importCmd) loadJekyllConfig(fs afero.Fs, jekyllRoot string) map[string]
 	return c
 }
 
-func (i *importCmd) createConfigFromJekyll(fs afero.Fs, inpath string, kind string, jekyllConfig map[string]interface{}) (err error) {
+func (i *importCmd) createConfigFromJekyll(fs afero.Fs, inpath string, kind metadecoders.Format, jekyllConfig map[string]interface{}) (err error) {
 	title := "My New Hugo Site"
 	baseURL := "http://example.org/"
 
@@ -288,77 +293,20 @@ func (i *importCmd) createConfigFromJekyll(fs afero.Fs, inpath string, kind stri
 		"languageCode":       "en-us",
 		"disablePathToLower": true,
 	}
-	kind = parser.FormatSanitize(kind)
 
 	var buf bytes.Buffer
-	err = parser.InterfaceToConfig(in, parser.FormatToLeadRune(kind), &buf)
+	err = parser.InterfaceToConfig(in, kind, &buf)
 	if err != nil {
 		return err
 	}
 
-	return helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), &buf, fs)
-}
-
-func copyFile(source string, dest string) error {
-	sf, err := os.Open(source)
-	if err != nil {
-		return err
-	}
-	defer sf.Close()
-	df, err := os.Create(dest)
-	if err != nil {
-		return err
-	}
-	defer df.Close()
-	_, err = io.Copy(df, sf)
-	if err == nil {
-		si, err := os.Stat(source)
-		if err != nil {
-			err = os.Chmod(dest, si.Mode())
-
-			if err != nil {
-				return err
-			}
-		}
-
-	}
-	return nil
-}
-
-func copyDir(source string, dest string) error {
-	fi, err := os.Stat(source)
-	if err != nil {
-		return err
-	}
-	if !fi.IsDir() {
-		return errors.New(source + " is not a directory")
-	}
-	err = os.MkdirAll(dest, fi.Mode())
-	if err != nil {
-		return err
-	}
-	entries, err := ioutil.ReadDir(source)
-	for _, entry := range entries {
-		sfp := filepath.Join(source, entry.Name())
-		dfp := filepath.Join(dest, entry.Name())
-		if entry.IsDir() {
-			err = copyDir(sfp, dfp)
-			if err != nil {
-				jww.ERROR.Println(err)
-			}
-		} else {
-			err = copyFile(sfp, dfp)
-			if err != nil {
-				jww.ERROR.Println(err)
-			}
-		}
-
-	}
-	return nil
+	return helpers.WriteToDisk(filepath.Join(inpath, "config."+string(kind)), &buf, fs)
 }
 
 func (i *importCmd) copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyllPostDirs map[string]bool) (err error) {
-	fi, err := os.Stat(jekyllRoot)
+	fs := hugofs.Os
+
+	fi, err := fs.Stat(jekyllRoot)
 	if err != nil {
 		return err
 	}
@@ -370,13 +318,17 @@ func (i *importCmd) copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyllPos
 		return err
 	}
 	entries, err := ioutil.ReadDir(jekyllRoot)
+	if err != nil {
+		return err
+	}
+
 	for _, entry := range entries {
 		sfp := filepath.Join(jekyllRoot, entry.Name())
 		dfp := filepath.Join(dest, entry.Name())
 		if entry.IsDir() {
 			if entry.Name()[0] != '_' && entry.Name()[0] != '.' {
 				if _, ok := jekyllPostDirs[entry.Name()]; !ok {
-					err = copyDir(sfp, dfp)
+					err = hugio.CopyDir(fs, sfp, dfp, nil)
 					if err != nil {
 						jww.ERROR.Println(err)
 					}
@@ -395,7 +347,7 @@ func (i *importCmd) copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyllPos
 			}
 
 			if !isExcept && entry.Name()[0] != '.' && entry.Name()[0] != '_' {
-				err = copyFile(sfp, dfp)
+				err = hugio.CopyFile(fs, sfp, dfp)
 				if err != nil {
 					jww.ERROR.Println(err)
 				}
@@ -445,39 +397,25 @@ func convertJekyllPost(s *hugolib.Site, path, relPath, targetDir string, draft b
 		return err
 	}
 
-	psr, err := parser.ReadFrom(bytes.NewReader(contentBytes))
+	pf, err := parseContentFile(bytes.NewReader(contentBytes))
 	if err != nil {
 		jww.ERROR.Println("Parse file error:", path)
 		return err
 	}
 
-	metadata, err := psr.Metadata()
-	if err != nil {
-		jww.ERROR.Println("Processing file error:", path)
-		return err
-	}
-
-	newmetadata, err := convertJekyllMetaData(metadata, postName, postDate, draft)
+	newmetadata, err := convertJekyllMetaData(pf.frontMatter, postName, postDate, draft)
 	if err != nil {
 		jww.ERROR.Println("Convert metadata error:", path)
 		return err
 	}
 
-	jww.TRACE.Println(newmetadata)
-	content := convertJekyllContent(newmetadata, string(psr.Content()))
+	content := convertJekyllContent(newmetadata, string(pf.content))
 
-	page, err := s.NewPage(filename)
-	if err != nil {
-		jww.ERROR.Println("New page error", filename)
-		return err
+	fs := hugofs.Os
+	if err := helpers.WriteToDisk(targetFile, strings.NewReader(content), fs); err != nil {
+		return fmt.Errorf("failed to save file %q: %s", filename, err)
 	}
 
-	page.SetSourceContent([]byte(content))
-	page.SetSourceMetaData(newmetadata, parser.FormatToLeadRune("yaml"))
-	page.SaveSourceAs(targetFile)
-
-	jww.TRACE.Println("Target file:", targetFile)
-
 	return nil
 }
 
@@ -557,7 +495,6 @@ func convertJekyllContent(m interface{}, content string) string {
 	}{
 		{regexp.MustCompile("(?i)<!-- more -->"), "<!--more-->"},
 		{regexp.MustCompile(`\{%\s*raw\s*%\}\s*(.*?)\s*\{%\s*endraw\s*%\}`), "$1"},
-		{regexp.MustCompile(`{%\s*highlight\s*(.*?)\s*%}`), "{{< highlight $1 >}}"},
 		{regexp.MustCompile(`{%\s*endhighlight\s*%}`), "{{< / highlight >}}"},
 	}
 
@@ -571,6 +508,7 @@ func convertJekyllContent(m interface{}, content string) string {
 	}{
 		// Octopress image tag: http://octopress.org/docs/plugins/image-tag/
 		{regexp.MustCompile(`{%\s+img\s*(.*?)\s*%}`), replaceImageTag},
+		{regexp.MustCompile(`{%\s*highlight\s*(.*?)\s*%}`), replaceHighlightTag},
 	}
 
 	for _, replace := range replaceListFunc {
@@ -580,6 +518,50 @@ func convertJekyllContent(m interface{}, content string) string {
 	return content
 }
 
+func replaceHighlightTag(match string) string {
+	r := regexp.MustCompile(`{%\s*highlight\s*(.*?)\s*%}`)
+	parts := r.FindStringSubmatch(match)
+	lastQuote := rune(0)
+	f := func(c rune) bool {
+		switch {
+		case c == lastQuote:
+			lastQuote = rune(0)
+			return false
+		case lastQuote != rune(0):
+			return false
+		case unicode.In(c, unicode.Quotation_Mark):
+			lastQuote = c
+			return false
+		default:
+			return unicode.IsSpace(c)
+		}
+	}
+	// splitting string by space but considering quoted section
+	items := strings.FieldsFunc(parts[1], f)
+
+	result := bytes.NewBufferString("{{< highlight ")
+	result.WriteString(items[0]) // language
+	options := items[1:]
+	for i, opt := range options {
+		opt = strings.Replace(opt, "\"", "", -1)
+		if opt == "linenos" {
+			opt = "linenos=table"
+		}
+		if i == 0 {
+			opt = " \"" + opt
+		}
+		if i < len(options)-1 {
+			opt += ","
+		} else if i == len(options)-1 {
+			opt += "\""
+		}
+		result.WriteString(opt)
+	}
+
+	result.WriteString(" >}}")
+	return result.String()
+}
+
 func replaceImageTag(match string) string {
 	r := regexp.MustCompile(`{%\s+img\s*(\p{L}*)\s+([\S]*/[\S]+)\s+(\d*)\s*(\d*)\s*(.*?)\s*%}`)
 	result := bytes.NewBufferString("{{< figure ")

commands/import_jekyll_test.go

@@ -15,12 +15,14 @@ package commands
 
 import (
 	"encoding/json"
-	"github.com/stretchr/testify/assert"
 	"testing"
 	"time"
+
+	qt "github.com/frankban/quicktest"
 )
 
 func TestParseJekyllFilename(t *testing.T) {
+	c := qt.New(t)
 	filenameArray := []string{
 		"2015-01-02-test.md",
 		"2012-03-15-中文.markup",
@@ -36,13 +38,14 @@ func TestParseJekyllFilename(t *testing.T) {
 
 	for i, filename := range filenameArray {
 		postDate, postName, err := parseJekyllFilename(filename)
-		assert.Equal(t, err, nil)
-		assert.Equal(t, expectResult[i].postDate.Format("2006-01-02"), postDate.Format("2006-01-02"))
-		assert.Equal(t, expectResult[i].postName, postName)
+		c.Assert(err, qt.IsNil)
+		c.Assert(expectResult[i].postDate.Format("2006-01-02"), qt.Equals, postDate.Format("2006-01-02"))
+		c.Assert(expectResult[i].postName, qt.Equals, postName)
 	}
 }
 
 func TestConvertJekyllMetadata(t *testing.T) {
+	c := qt.New(t)
 	testDataList := []struct {
 		metadata interface{}
 		postName string
@@ -73,14 +76,15 @@ func TestConvertJekyllMetadata(t *testing.T) {
 
 	for _, data := range testDataList {
 		result, err := convertJekyllMetaData(data.metadata, data.postName, data.postDate, data.draft)
-		assert.Equal(t, nil, err)
+		c.Assert(err, qt.IsNil)
 		jsonResult, err := json.Marshal(result)
-		assert.Equal(t, nil, err)
-		assert.Equal(t, data.expect, string(jsonResult))
+		c.Assert(err, qt.IsNil)
+		c.Assert(string(jsonResult), qt.Equals, data.expect)
 	}
 }
 
 func TestConvertJekyllContent(t *testing.T) {
+	c := qt.New(t)
 	testDataList := []struct {
 		metadata interface{}
 		content  string
@@ -97,6 +101,9 @@ func TestConvertJekyllContent(t *testing.T) {
 		{map[interface{}]interface{}{},
 			"{% highlight go %}\nvar s int\n{% endhighlight %}",
 			"{{< highlight go >}}\nvar s int\n{{< / highlight >}}"},
+		{map[interface{}]interface{}{},
+			"{% highlight go linenos hl_lines=\"1 2\" %}\nvar s string\nvar i int\n{% endhighlight %}",
+			"{{< highlight go \"linenos=table,hl_lines=1 2\" >}}\nvar s string\nvar i int\n{{< / highlight >}}"},
 
 		// Octopress image tag
 		{map[interface{}]interface{}{},
@@ -121,6 +128,6 @@ func TestConvertJekyllContent(t *testing.T) {
 
 	for _, data := range testDataList {
 		result := convertJekyllContent(data.metadata, data.content)
-		assert.Equal(t, data.expect, result)
+		c.Assert(data.expect, qt.Equals, result)
 	}
 }

commands/limit_darwin.go

@@ -36,14 +36,17 @@ This is primarily to ensure that Hugo can watch enough files on some OSs`,
 			var rLimit syscall.Rlimit
 			err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
 			if err != nil {
-				return newSystemError("Error Getting Rlimit ", err)
+				return newSystemError("Error Getting rlimit ", err)
 			}
 
 			jww.FEEDBACK.Println("Current rLimit:", rLimit)
 
+			if rLimit.Cur >= newRlimit {
+				return nil
+			}
+
 			jww.FEEDBACK.Println("Attempting to increase limit")
-			rLimit.Max = 999999
-			rLimit.Cur = 999999
+			rLimit.Cur = newRlimit
 			err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
 			if err != nil {
 				return newSystemError("Error Setting rLimit ", err)
@@ -61,18 +64,21 @@ This is primarily to ensure that Hugo can watch enough files on some OSs`,
 	return &limitCmd{baseCmd: newBaseCmd(ccmd)}
 }
 
+const newRlimit = 10240
+
 func tweakLimit() {
 	var rLimit syscall.Rlimit
 	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
 	if err != nil {
-		jww.ERROR.Println("Unable to obtain rLimit", err)
+		jww.WARN.Println("Unable to get rlimit:", err)
+		return
 	}
-	if rLimit.Cur < rLimit.Max {
-		rLimit.Max = 64000
-		rLimit.Cur = 64000
+	if rLimit.Cur < newRlimit {
+		rLimit.Cur = newRlimit
 		err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
 		if err != nil {
-			jww.WARN.Println("Unable to increase number of open files limit", err)
+			// This may not succeed, see https://github.com/golang/go/issues/30401
+			jww.INFO.Println("Unable to increase number of open files limit:", err)
 		}
 	}
 }

commands/list.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,9 +14,14 @@
 package commands
 
 import (
-	"path/filepath"
+	"encoding/csv"
+	"os"
+	"strconv"
+	"strings"
+	"time"
 
 	"github.com/gohugoio/hugo/hugolib"
+	"github.com/gohugoio/hugo/resources/resource"
 	"github.com/spf13/cobra"
 	jww "github.com/spf13/jwalterweatherman"
 )
@@ -28,6 +33,32 @@ type listCmd struct {
 	*baseCmd
 }
 
+func (lc *listCmd) buildSites(config map[string]interface{}) (*hugolib.HugoSites, error) {
+	cfgInit := func(c *commandeer) error {
+		for key, value := range config {
+			c.Set(key, value)
+		}
+		return nil
+	}
+
+	c, err := initializeConfig(true, false, &lc.hugoBuilderCommon, lc, cfgInit)
+	if err != nil {
+		return nil, err
+	}
+
+	sites, err := hugolib.NewHugoSites(*c.DepsCfg)
+
+	if err != nil {
+		return nil, newSystemError("Error creating sites", err)
+	}
+
+	if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
+		return nil, newSystemError("Error Processing Source Content", err)
+	}
+
+	return sites, nil
+}
+
 func newListCmd() *listCmd {
 	cc := &listCmd{}
 
@@ -46,106 +77,127 @@ List requires a subcommand, e.g. ` + "`hugo list drafts`.",
 			Short: "List all drafts",
 			Long:  `List all of the drafts in your content directory.`,
 			RunE: func(cmd *cobra.Command, args []string) error {
-				cfgInit := func(c *commandeer) error {
-					c.Set("buildDrafts", true)
-					return nil
-				}
-				c, err := initializeConfig(false, &cc.hugoBuilderCommon, cc, cfgInit)
-				if err != nil {
-					return err
-				}
-
-				sites, err := hugolib.NewHugoSites(*c.DepsCfg)
+				sites, err := cc.buildSites(map[string]interface{}{"buildDrafts": true})
 
 				if err != nil {
-					return newSystemError("Error creating sites", err)
-				}
-
-				if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
-					return newSystemError("Error Processing Source Content", err)
+					return newSystemError("Error building sites", err)
 				}
 
 				for _, p := range sites.Pages() {
-					if p.IsDraft() {
-						jww.FEEDBACK.Println(filepath.Join(p.File.Dir(), p.File.LogicalName()))
+					if p.Draft() {
+						jww.FEEDBACK.Println(strings.TrimPrefix(p.File().Filename(), sites.WorkingDir+string(os.PathSeparator)))
 					}
-
 				}
 
 				return nil
-
 			},
 		},
 		&cobra.Command{
 			Use:   "future",
 			Short: "List all posts dated in the future",
-			Long: `List all of the posts in your content directory which will be
-posted in the future.`,
+			Long:  `List all of the posts in your content directory which will be posted in the future.`,
 			RunE: func(cmd *cobra.Command, args []string) error {
-				cfgInit := func(c *commandeer) error {
-					c.Set("buildFuture", true)
-					return nil
-				}
-				c, err := initializeConfig(false, &cc.hugoBuilderCommon, cc, cfgInit)
-				if err != nil {
-					return err
-				}
-
-				sites, err := hugolib.NewHugoSites(*c.DepsCfg)
+				sites, err := cc.buildSites(map[string]interface{}{"buildFuture": true})
 
 				if err != nil {
-					return newSystemError("Error creating sites", err)
+					return newSystemError("Error building sites", err)
 				}
 
-				if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
-					return newSystemError("Error Processing Source Content", err)
-				}
+				writer := csv.NewWriter(os.Stdout)
+				defer writer.Flush()
 
 				for _, p := range sites.Pages() {
-					if p.IsFuture() {
-						jww.FEEDBACK.Println(filepath.Join(p.File.Dir(), p.File.LogicalName()))
+					if resource.IsFuture(p) {
+						err := writer.Write([]string{
+							strings.TrimPrefix(p.File().Filename(), sites.WorkingDir+string(os.PathSeparator)),
+							p.PublishDate().Format(time.RFC3339),
+						})
+						if err != nil {
+							return newSystemError("Error writing future posts to stdout", err)
+						}
 					}
-
 				}
 
 				return nil
-
 			},
 		},
 		&cobra.Command{
 			Use:   "expired",
 			Short: "List all posts already expired",
-			Long: `List all of the posts in your content directory which has already
-expired.`,
+			Long:  `List all of the posts in your content directory which has already expired.`,
 			RunE: func(cmd *cobra.Command, args []string) error {
-				cfgInit := func(c *commandeer) error {
-					c.Set("buildExpired", true)
-					return nil
-				}
-				c, err := initializeConfig(false, &cc.hugoBuilderCommon, cc, cfgInit)
-				if err != nil {
-					return err
-				}
-
-				sites, err := hugolib.NewHugoSites(*c.DepsCfg)
+				sites, err := cc.buildSites(map[string]interface{}{"buildExpired": true})
 
 				if err != nil {
-					return newSystemError("Error creating sites", err)
+					return newSystemError("Error building sites", err)
 				}
 
-				if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
-					return newSystemError("Error Processing Source Content", err)
-				}
+				writer := csv.NewWriter(os.Stdout)
+				defer writer.Flush()
 
 				for _, p := range sites.Pages() {
-					if p.IsExpired() {
-						jww.FEEDBACK.Println(filepath.Join(p.File.Dir(), p.File.LogicalName()))
+					if resource.IsExpired(p) {
+						err := writer.Write([]string{
+							strings.TrimPrefix(p.File().Filename(), sites.WorkingDir+string(os.PathSeparator)),
+							p.ExpiryDate().Format(time.RFC3339),
+						})
+						if err != nil {
+							return newSystemError("Error writing expired posts to stdout", err)
+						}
 					}
-
 				}
 
 				return nil
+			},
+		},
+		&cobra.Command{
+			Use:   "all",
+			Short: "List all posts",
+			Long:  `List all of the posts in your content directory, include drafts, future and expired pages.`,
+			RunE: func(cmd *cobra.Command, args []string) error {
+				sites, err := cc.buildSites(map[string]interface{}{
+					"buildExpired": true,
+					"buildDrafts":  true,
+					"buildFuture":  true,
+				})
 
+				if err != nil {
+					return newSystemError("Error building sites", err)
+				}
+
+				writer := csv.NewWriter(os.Stdout)
+				defer writer.Flush()
+
+				writer.Write([]string{
+					"path",
+					"slug",
+					"title",
+					"date",
+					"expiryDate",
+					"publishDate",
+					"draft",
+					"permalink",
+				})
+				for _, p := range sites.Pages() {
+					if !p.IsPage() {
+						continue
+					}
+					err := writer.Write([]string{
+						strings.TrimPrefix(p.File().Filename(), sites.WorkingDir+string(os.PathSeparator)),
+						p.Slug(),
+						p.Title(),
+						p.Date().Format(time.RFC3339),
+						p.ExpiryDate().Format(time.RFC3339),
+						p.PublishDate().Format(time.RFC3339),
+						strconv.FormatBool(p.Draft()),
+						p.Permalink(),
+					})
+					if err != nil {
+						return newSystemError("Error writing posts to stdout", err)
+					}
+				}
+
+				return nil
 			},
 		},
 	)

commands/list_test.go

@@ -0,0 +1,72 @@
+package commands
+
+import (
+	"bytes"
+	"encoding/csv"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+	"github.com/spf13/cobra"
+)
+
+func captureStdout(f func() (*cobra.Command, error)) (string, error) {
+	old := os.Stdout
+	r, w, _ := os.Pipe()
+	os.Stdout = w
+
+	_, err := f()
+
+	if err != nil {
+		return "", err
+	}
+
+	w.Close()
+	os.Stdout = old
+
+	var buf bytes.Buffer
+	io.Copy(&buf, r)
+	return buf.String(), nil
+}
+
+func TestListAll(t *testing.T) {
+	c := qt.New(t)
+	dir, err := createSimpleTestSite(t, testSiteConfig{})
+
+	c.Assert(err, qt.IsNil)
+
+	hugoCmd := newCommandsBuilder().addAll().build()
+	cmd := hugoCmd.getCommand()
+
+	defer func() {
+		os.RemoveAll(dir)
+	}()
+
+	cmd.SetArgs([]string{"-s=" + dir, "list", "all"})
+
+	out, err := captureStdout(cmd.ExecuteC)
+	c.Assert(err, qt.IsNil)
+
+	r := csv.NewReader(strings.NewReader(out))
+
+	header, err := r.Read()
+
+	c.Assert(err, qt.IsNil)
+	c.Assert(header, qt.DeepEquals, []string{
+		"path", "slug", "title",
+		"date", "expiryDate", "publishDate",
+		"draft", "permalink",
+	})
+
+	record, err := r.Read()
+
+	c.Assert(err, qt.IsNil)
+	c.Assert(record, qt.DeepEquals, []string{
+		filepath.Join("content", "p1.md"), "", "P1",
+		"0001-01-01T00:00:00Z", "0001-01-01T00:00:00Z", "0001-01-01T00:00:00Z",
+		"false", "https://example.org/p1/",
+	})
+}

commands/mod.go

@@ -0,0 +1,188 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package commands
+
+import (
+	"os"
+
+	"github.com/gohugoio/hugo/modules"
+	"github.com/spf13/cobra"
+)
+
+var _ cmder = (*modCmd)(nil)
+
+type modCmd struct {
+	*baseBuilderCmd
+}
+
+func (b *commandsBuilder) newModCmd() *modCmd {
+	c := &modCmd{}
+
+	const commonUsage = `
+Note that Hugo will always start out by resolving the components defined in the site
+configuration, provided by a _vendor directory (if no --ignoreVendor flag provided),
+Go Modules, or a folder inside the themes directory, in that order.
+
+See https://gohugo.io/hugo-modules/ for more information.
+
+`
+
+	cmd := &cobra.Command{
+		Use:   "mod",
+		Short: "Various Hugo Modules helpers.",
+		Long: `Various helpers to help manage the modules in your project's dependency graph.
+
+Most operations here requires a Go version installed on your system (>= Go 1.12) and the relevant VCS client (typically Git).
+This is not needed if you only operate on modules inside /themes or if you have vendored them via "hugo mod vendor".
+
+` + commonUsage,
+
+		RunE: nil,
+	}
+
+	cmd.AddCommand(
+		&cobra.Command{
+			Use:                "get",
+			DisableFlagParsing: true,
+			Short:              "Resolves dependencies in your current Hugo Project.",
+			Long: `
+Resolves dependencies in your current Hugo Project.
+
+Some examples:
+
+Install the latest version possible for a given module:
+
+    hugo mod get github.com/gohugoio/testshortcodes
+    
+Install a specific version:
+
+    hugo mod get github.com/gohugoio/testshortcodes@v0.3.0
+
+Install the latest versions of all module dependencies:
+
+    hugo mod get -u
+
+Run "go help get" for more information. All flags available for "go get" is also relevant here.
+` + commonUsage,
+			RunE: func(cmd *cobra.Command, args []string) error {
+				return c.withModsClient(false, func(c *modules.Client) error {
+					// We currently just pass on the flags we get to Go and
+					// need to do the flag handling manually.
+					if len(args) == 1 && args[0] == "-h" {
+						return cmd.Help()
+					}
+					return c.Get(args...)
+				})
+			},
+		},
+		&cobra.Command{
+			Use:   "graph",
+			Short: "Print a module dependency graph.",
+			Long: `Print a module dependency graph with information about module status (disabled, vendored).
+Note that for vendored modules, that is the version listed and not the one from go.mod.
+`,
+			RunE: func(cmd *cobra.Command, args []string) error {
+				return c.withModsClient(true, func(c *modules.Client) error {
+					return c.Graph(os.Stdout)
+				})
+			},
+		},
+		&cobra.Command{
+			Use:   "init",
+			Short: "Initialize this project as a Hugo Module.",
+			Long: `Initialize this project as a Hugo Module.
+It will try to guess the module path, but you may help by passing it as an argument, e.g:
+
+    hugo mod init github.com/gohugoio/testshortcodes
+
+Note that Hugo Modules supports multi-module projects, so you can initialize a Hugo Module
+inside a subfolder on GitHub, as one example.
+`,
+			RunE: func(cmd *cobra.Command, args []string) error {
+				var path string
+				if len(args) >= 1 {
+					path = args[0]
+				}
+				return c.withModsClient(false, func(c *modules.Client) error {
+					return c.Init(path)
+				})
+			},
+		},
+		&cobra.Command{
+			Use:   "vendor",
+			Short: "Vendor all module dependencies into the _vendor directory.",
+			Long: `Vendor all module dependencies into the _vendor directory.
+
+If a module is vendored, that is where Hugo will look for it's dependencies.
+`,
+			RunE: func(cmd *cobra.Command, args []string) error {
+				return c.withModsClient(true, func(c *modules.Client) error {
+					return c.Vendor()
+				})
+			},
+		},
+		&cobra.Command{
+			Use:   "tidy",
+			Short: "Remove unused entries in go.mod and go.sum.",
+			RunE: func(cmd *cobra.Command, args []string) error {
+				return c.withModsClient(true, func(c *modules.Client) error {
+					return c.Tidy()
+				})
+			},
+		},
+		&cobra.Command{
+			Use:   "clean",
+			Short: "Delete the entire Hugo Module cache.",
+			Long: `Delete the entire Hugo Module cache.
+
+Note that after you run this command, all of your dependencies will be re-downloaded next time you run "hugo".
+
+Also note that if you configure a positive maxAge for the "modules" file cache, it will also be cleaned as part of "hugo --gc".
+ 
+`,
+			RunE: func(cmd *cobra.Command, args []string) error {
+				com, err := c.initConfig(true)
+				if err != nil {
+					return err
+				}
+
+				_, err = com.hugo().FileCaches.ModulesCache().Prune(true)
+				return err
+
+			},
+		},
+	)
+
+	c.baseBuilderCmd = b.newBuilderCmd(cmd)
+
+	return c
+
+}
+
+func (c *modCmd) withModsClient(failOnMissingConfig bool, f func(*modules.Client) error) error {
+	com, err := c.initConfig(failOnMissingConfig)
+	if err != nil {
+		return err
+	}
+
+	return f(com.hugo().ModulesClient)
+}
+
+func (c *modCmd) initConfig(failOnNoConfig bool) (*commandeer, error) {
+	com, err := initializeConfig(failOnNoConfig, false, &c.hugoBuilderCommon, c, nil)
+	if err != nil {
+		return nil, err
+	}
+	return com, nil
+}

commands/new.go

@@ -30,16 +30,14 @@ import (
 var _ cmder = (*newCmd)(nil)
 
 type newCmd struct {
-	hugoBuilderCommon
 	contentEditor string
 	contentType   string
 
-	*baseCmd
+	*baseBuilderCmd
 }
 
-func newNewCmd() *newCmd {
-	cc := &newCmd{}
-	cc.baseCmd = newBaseCmd(&cobra.Command{
+func (b *commandsBuilder) newNewCmd() *newCmd {
+	cmd := &cobra.Command{
 		Use:   "new [path]",
 		Short: "Create new content for your site",
 		Long: `Create a new content file and automatically set the date and title.
@@ -47,18 +45,20 @@ It will guess which kind of file to create based on the path provided.
 
 You can also specify the kind with ` + "`-k KIND`" + `.
 
-If archetypes are provided in your theme or site, they will be used.`,
+If archetypes are provided in your theme or site, they will be used.
 
-		RunE: cc.newContent,
-	})
+Ensure you run this within the root directory of your site.`,
+	}
 
-	cc.cmd.Flags().StringVarP(&cc.contentType, "kind", "k", "", "content type to create")
-	cc.cmd.PersistentFlags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")
-	cc.cmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{})
-	cc.cmd.Flags().StringVar(&cc.contentEditor, "editor", "", "edit new content with this editor, if provided")
+	cc := &newCmd{baseBuilderCmd: b.newBuilderCmd(cmd)}
 
-	cc.cmd.AddCommand(newNewSiteCmd().getCommand())
-	cc.cmd.AddCommand(newNewThemeCmd().getCommand())
+	cmd.Flags().StringVarP(&cc.contentType, "kind", "k", "", "content type to create")
+	cmd.Flags().StringVar(&cc.contentEditor, "editor", "", "edit new content with this editor, if provided")
+
+	cmd.AddCommand(newNewSiteCmd().getCommand())
+	cmd.AddCommand(newNewThemeCmd().getCommand())
+
+	cmd.RunE = cc.newContent
 
 	return cc
 }
@@ -71,7 +71,7 @@ func (n *newCmd) newContent(cmd *cobra.Command, args []string) error {
 		return nil
 	}
 
-	c, err := initializeConfig(false, &n.hugoBuilderCommon, n, cfgInit)
+	c, err := initializeConfig(true, false, &n.hugoBuilderCommon, n, cfgInit)
 
 	if err != nil {
 		return err
@@ -85,69 +85,13 @@ func (n *newCmd) newContent(cmd *cobra.Command, args []string) error {
 
 	var kind string
 
-	createPath, kind = newContentPathSection(createPath)
+	createPath, kind = newContentPathSection(c.hugo(), createPath)
 
 	if n.contentType != "" {
 		kind = n.contentType
 	}
 
-	cfg := c.DepsCfg
-
-	ps, err := helpers.NewPathSpec(cfg.Fs, cfg.Cfg)
-	if err != nil {
-		return err
-	}
-
-	// If a site isn't in use in the archetype template, we can skip the build.
-	siteFactory := func(filename string, siteUsed bool) (*hugolib.Site, error) {
-		if !siteUsed {
-			return hugolib.NewSite(*cfg)
-		}
-		var s *hugolib.Site
-		if err := c.initSites(); err != nil {
-			return nil, err
-		}
-
-		if err := c.hugo.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
-			return nil, err
-		}
-
-		s = c.hugo.Sites[0]
-
-		if len(c.hugo.Sites) > 1 {
-			// Find the best match.
-			for _, ss := range c.hugo.Sites {
-				if strings.Contains(createPath, "."+ss.Language.Lang) {
-					s = ss
-					break
-				}
-			}
-		}
-		return s, nil
-	}
-
-	return create.NewContent(ps, siteFactory, kind, createPath)
-}
-
-func nextStepsText() string {
-	var nextStepsText bytes.Buffer
-
-	nextStepsText.WriteString(`Just a few more steps and you're ready to go:
-
-1. Download a theme into the same-named folder.
-   Choose a theme from https://themes.gohugo.io/, or
-   create your own with the "hugo new theme <THEMENAME>" command.
-2. Perhaps you want to add some content. You can add single files
-   with "hugo new `)
-
-	nextStepsText.WriteString(filepath.Join("<SECTIONNAME>", "<FILENAME>.<FORMAT>"))
-
-	nextStepsText.WriteString(`".
-3. Start the built-in live server via "hugo server".
-
-Visit https://gohugo.io/ for quickstart guide and full documentation.`)
-
-	return nextStepsText.String()
+	return create.NewContent(c.hugo(), kind, createPath)
 }
 
 func mkdir(x ...string) {
@@ -168,10 +112,17 @@ func touchFile(fs afero.Fs, x ...string) {
 	}
 }
 
-func newContentPathSection(path string) (string, string) {
+func newContentPathSection(h *hugolib.HugoSites, path string) (string, string) {
 	// Forward slashes is used in all examples. Convert if needed.
 	// Issue #1133
 	createpath := filepath.FromSlash(path)
+
+	if h != nil {
+		for _, dir := range h.BaseFs.Content.Dirs {
+			createpath = strings.TrimPrefix(createpath, dir.Meta().Filename())
+		}
+	}
+
 	var section string
 	// assume the first directory is the section (kind)
 	if strings.Contains(createpath[1:], helpers.FilePathSeparator) {

commands/new_content_test.go

@@ -1,4 +1,4 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -19,19 +19,20 @@ import (
 
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/spf13/viper"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
+
+	qt "github.com/frankban/quicktest"
 )
 
 // Issue #1133
 func TestNewContentPathSectionWithForwardSlashes(t *testing.T) {
-	p, s := newContentPathSection("/post/new.md")
-	assert.Equal(t, filepath.FromSlash("/post/new.md"), p)
-	assert.Equal(t, "post", s)
+	c := qt.New(t)
+	p, s := newContentPathSection(nil, "/post/new.md")
+	c.Assert(p, qt.Equals, filepath.FromSlash("/post/new.md"))
+	c.Assert(s, qt.Equals, "post")
 }
 
 func checkNewSiteInited(fs *hugofs.Fs, basepath string, t *testing.T) {
-
+	c := qt.New(t)
 	paths := []string{
 		filepath.Join(basepath, "layouts"),
 		filepath.Join(basepath, "content"),
@@ -43,77 +44,82 @@ func checkNewSiteInited(fs *hugofs.Fs, basepath string, t *testing.T) {
 
 	for _, path := range paths {
 		_, err := fs.Source.Stat(path)
-		require.NoError(t, err)
+		c.Assert(err, qt.IsNil)
 	}
 }
 
 func TestDoNewSite(t *testing.T) {
+	c := qt.New(t)
 	n := newNewSiteCmd()
 	basepath := filepath.Join("base", "blog")
 	_, fs := newTestCfg()
 
-	require.NoError(t, n.doNewSite(fs, basepath, false))
+	c.Assert(n.doNewSite(fs, basepath, false), qt.IsNil)
 
 	checkNewSiteInited(fs, basepath, t)
 }
 
 func TestDoNewSite_noerror_base_exists_but_empty(t *testing.T) {
+	c := qt.New(t)
 	basepath := filepath.Join("base", "blog")
 	_, fs := newTestCfg()
 	n := newNewSiteCmd()
 
-	require.NoError(t, fs.Source.MkdirAll(basepath, 777))
+	c.Assert(fs.Source.MkdirAll(basepath, 0777), qt.IsNil)
 
-	require.NoError(t, n.doNewSite(fs, basepath, false))
+	c.Assert(n.doNewSite(fs, basepath, false), qt.IsNil)
 }
 
 func TestDoNewSite_error_base_exists(t *testing.T) {
+	c := qt.New(t)
 	basepath := filepath.Join("base", "blog")
 	_, fs := newTestCfg()
 	n := newNewSiteCmd()
 
-	require.NoError(t, fs.Source.MkdirAll(basepath, 777))
+	c.Assert(fs.Source.MkdirAll(basepath, 0777), qt.IsNil)
 	_, err := fs.Source.Create(filepath.Join(basepath, "foo"))
-	require.NoError(t, err)
+	c.Assert(err, qt.IsNil)
 	// Since the directory already exists and isn't empty, expect an error
-	require.Error(t, n.doNewSite(fs, basepath, false))
+	c.Assert(n.doNewSite(fs, basepath, false), qt.Not(qt.IsNil))
 
 }
 
 func TestDoNewSite_force_empty_dir(t *testing.T) {
+	c := qt.New(t)
 	basepath := filepath.Join("base", "blog")
 	_, fs := newTestCfg()
 	n := newNewSiteCmd()
 
-	require.NoError(t, fs.Source.MkdirAll(basepath, 777))
-
-	require.NoError(t, n.doNewSite(fs, basepath, true))
+	c.Assert(fs.Source.MkdirAll(basepath, 0777), qt.IsNil)
+	c.Assert(n.doNewSite(fs, basepath, true), qt.IsNil)
 
 	checkNewSiteInited(fs, basepath, t)
 }
 
 func TestDoNewSite_error_force_dir_inside_exists(t *testing.T) {
+	c := qt.New(t)
 	basepath := filepath.Join("base", "blog")
 	_, fs := newTestCfg()
 	n := newNewSiteCmd()
 
 	contentPath := filepath.Join(basepath, "content")
 
-	require.NoError(t, fs.Source.MkdirAll(contentPath, 777))
-	require.Error(t, n.doNewSite(fs, basepath, true))
+	c.Assert(fs.Source.MkdirAll(contentPath, 0777), qt.IsNil)
+	c.Assert(n.doNewSite(fs, basepath, true), qt.Not(qt.IsNil))
 }
 
 func TestDoNewSite_error_force_config_inside_exists(t *testing.T) {
+	c := qt.New(t)
 	basepath := filepath.Join("base", "blog")
 	_, fs := newTestCfg()
 	n := newNewSiteCmd()
 
 	configPath := filepath.Join(basepath, "config.toml")
-	require.NoError(t, fs.Source.MkdirAll(basepath, 777))
+	c.Assert(fs.Source.MkdirAll(basepath, 0777), qt.IsNil)
 	_, err := fs.Source.Create(configPath)
-	require.NoError(t, err)
+	c.Assert(err, qt.IsNil)
 
-	require.Error(t, n.doNewSite(fs, basepath, true))
+	c.Assert(n.doNewSite(fs, basepath, true), qt.Not(qt.IsNil))
 }
 
 func newTestCfg() (*viper.Viper, *hugofs.Fs) {

commands/new_site.go

@@ -15,24 +15,21 @@ package commands
 
 import (
 	"bytes"
-	"path/filepath"
-
-	"github.com/spf13/viper"
-
 	"errors"
-
-	"github.com/gohugoio/hugo/create"
-
-	"fmt"
-
+	"path/filepath"
 	"strings"
 
-	"github.com/gohugoio/hugo/parser"
+	"github.com/gohugoio/hugo/parser/metadecoders"
 
+	_errors "github.com/pkg/errors"
+
+	"github.com/gohugoio/hugo/create"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
+	"github.com/gohugoio/hugo/parser"
 	"github.com/spf13/cobra"
 	jww "github.com/spf13/jwalterweatherman"
+	"github.com/spf13/viper"
 )
 
 var _ cmder = (*newSiteCmd)(nil)
@@ -98,13 +95,13 @@ func (n *newSiteCmd) doNewSite(fs *hugofs.Fs, basepath string, force bool) error
 
 	for _, dir := range dirs {
 		if err := fs.Source.MkdirAll(dir, 0777); err != nil {
-			return fmt.Errorf("Failed to create dir: %s", err)
+			return _errors.Wrap(err, "Failed to create dir")
 		}
 	}
 
 	createConfig(fs, basepath, n.configFormat)
 
-	// Create a defaul archetype file.
+	// Create a default archetype file.
 	helpers.SafeWriteToDisk(filepath.Join(archeTypePath, "default.md"),
 		strings.NewReader(create.ArchetypeTemplateTemplate), fs.Source)
 
@@ -136,13 +133,33 @@ func createConfig(fs *hugofs.Fs, inpath string, kind string) (err error) {
 		"title":        "My New Hugo Site",
 		"languageCode": "en-us",
 	}
-	kind = parser.FormatSanitize(kind)
 
 	var buf bytes.Buffer
-	err = parser.InterfaceToConfig(in, parser.FormatToLeadRune(kind), &buf)
+	err = parser.InterfaceToConfig(in, metadecoders.FormatFromString(kind), &buf)
 	if err != nil {
 		return err
 	}
 
 	return helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), &buf, fs.Source)
 }
+
+func nextStepsText() string {
+	var nextStepsText bytes.Buffer
+
+	nextStepsText.WriteString(`Just a few more steps and you're ready to go:
+
+1. Download a theme into the same-named folder.
+   Choose a theme from https://themes.gohugo.io/ or
+   create your own with the "hugo new theme <THEMENAME>" command.
+2. Perhaps you want to add some content. You can add single files
+   with "hugo new `)
+
+	nextStepsText.WriteString(filepath.Join("<SECTIONNAME>", "<FILENAME>.<FORMAT>"))
+
+	nextStepsText.WriteString(`".
+3. Start the built-in live server via "hugo server".
+
+Visit https://gohugo.io/ for quickstart guide and full documentation.`)
+
+	return nextStepsText.String()
+}

commands/new_theme.go

@@ -14,16 +14,14 @@
 package commands
 
 import (
-	"path/filepath"
-
 	"bytes"
-
+	"errors"
+	"path/filepath"
 	"strings"
 	"time"
 
-	"github.com/gohugoio/hugo/hugofs"
-
 	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/hugofs"
 	"github.com/spf13/cobra"
 	jww "github.com/spf13/jwalterweatherman"
 )
@@ -53,8 +51,9 @@ as you see fit.`,
 	return ccmd
 }
 
+// newTheme creates a new Hugo theme template
 func (n *newThemeCmd) newTheme(cmd *cobra.Command, args []string) error {
-	c, err := initializeConfig(false, &n.hugoBuilderCommon, n, nil)
+	c, err := initializeConfig(false, false, &n.hugoBuilderCommon, n, nil)
 
 	if err != nil {
 		return err
@@ -64,13 +63,13 @@ func (n *newThemeCmd) newTheme(cmd *cobra.Command, args []string) error {
 		return newUserError("theme name needs to be provided")
 	}
 
-	createpath := c.PathSpec().AbsPathify(filepath.Join(c.Cfg.GetString("themesDir"), args[0]))
-	jww.INFO.Println("creating theme at", createpath)
+	createpath := c.hugo().PathSpec.AbsPathify(filepath.Join(c.Cfg.GetString("themesDir"), args[0]))
+	jww.FEEDBACK.Println("Creating theme at", createpath)
 
 	cfg := c.DepsCfg
 
 	if x, _ := helpers.Exists(createpath, cfg.Fs.Source); x {
-		return newUserError(createpath, "already exists")
+		return errors.New(createpath + " already exists")
 	}
 
 	mkdir(createpath, "layouts", "_default")
@@ -81,6 +80,24 @@ func (n *newThemeCmd) newTheme(cmd *cobra.Command, args []string) error {
 	touchFile(cfg.Fs.Source, createpath, "layouts", "_default", "list.html")
 	touchFile(cfg.Fs.Source, createpath, "layouts", "_default", "single.html")
 
+	baseofDefault := []byte(`<!DOCTYPE html>
+<html>
+    {{- partial "head.html" . -}}
+    <body>
+        {{- partial "header.html" . -}}
+        <div id="content">
+        {{- block "main" . }}{{- end }}
+        </div>
+        {{- partial "footer.html" . -}}
+    </body>
+</html>
+`)
+	err = helpers.WriteToDisk(filepath.Join(createpath, "layouts", "_default", "baseof.html"), bytes.NewReader(baseofDefault), cfg.Fs.Source)
+	if err != nil {
+		return err
+	}
+
+	touchFile(cfg.Fs.Source, createpath, "layouts", "partials", "head.html")
 	touchFile(cfg.Fs.Source, createpath, "layouts", "partials", "header.html")
 	touchFile(cfg.Fs.Source, createpath, "layouts", "partials", "footer.html")
 
@@ -118,7 +135,7 @@ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 `)
 
-	err = helpers.WriteToDisk(filepath.Join(createpath, "LICENSE.md"), bytes.NewReader(by), cfg.Fs.Source)
+	err = helpers.WriteToDisk(filepath.Join(createpath, "LICENSE"), bytes.NewReader(by), cfg.Fs.Source)
 	if err != nil {
 		return err
 	}
@@ -135,12 +152,12 @@ func (n *newThemeCmd) createThemeMD(fs *hugofs.Fs, inpath string) (err error) {
 
 name = "` + strings.Title(helpers.MakeTitle(filepath.Base(inpath))) + `"
 license = "MIT"
-licenselink = "https://github.com/yourname/yourtheme/blob/master/LICENSE.md"
+licenselink = "https://github.com/yourname/yourtheme/blob/master/LICENSE"
 description = ""
 homepage = "http://example.com/"
 tags = []
 features = []
-min_version = "0.38"
+min_version = "0.41"
 
 [author]
   name = ""

commands/server.go

@@ -1,4 +1,4 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
 package commands
 
 import (
+	"bytes"
 	"fmt"
 	"net"
 	"net/http"
@@ -21,6 +22,7 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
@@ -28,10 +30,12 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/gohugoio/hugo/livereload"
+	"github.com/gohugoio/hugo/tpl"
 
 	"github.com/gohugoio/hugo/config"
-
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
@@ -52,7 +56,8 @@ type serverCmd struct {
 	serverWatch       bool
 	noHTTPCache       bool
 
-	disableFastRender bool
+	disableFastRender   bool
+	disableBrowserError bool
 
 	*baseBuilderCmd
 }
@@ -93,6 +98,7 @@ of a second, you will be able to save and see your changes nearly instantly.`,
 	cc.cmd.Flags().BoolVar(&cc.navigateToChanged, "navigateToChanged", false, "navigate to changed content file on live browser reload")
 	cc.cmd.Flags().BoolVar(&cc.renderToDisk, "renderToDisk", false, "render to Destination path (default is render to memory & serve from there)")
 	cc.cmd.Flags().BoolVar(&cc.disableFastRender, "disableFastRender", false, "enables full re-renders on changes")
+	cc.cmd.Flags().BoolVar(&cc.disableBrowserError, "disableBrowserError", false, "do not show build errors in the browser")
 
 	cc.cmd.Flags().String("memstats", "", "log memory usage to this file")
 	cc.cmd.Flags().String("meminterval", "100ms", "interval to poll memory usage (requires --memstats), valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\".")
@@ -122,27 +128,30 @@ func (f noDirFile) Readdir(count int) ([]os.FileInfo, error) {
 
 var serverPorts []int
 
-func (s *serverCmd) server(cmd *cobra.Command, args []string) error {
+func (sc *serverCmd) server(cmd *cobra.Command, args []string) error {
 	// If a Destination is provided via flag write to disk
 	destination, _ := cmd.Flags().GetString("destination")
 	if destination != "" {
-		s.renderToDisk = true
+		sc.renderToDisk = true
 	}
 
 	var serverCfgInit sync.Once
 
 	cfgInit := func(c *commandeer) error {
-		c.Set("renderToMemory", !s.renderToDisk)
+		c.Set("renderToMemory", !sc.renderToDisk)
 		if cmd.Flags().Changed("navigateToChanged") {
-			c.Set("navigateToChanged", s.navigateToChanged)
+			c.Set("navigateToChanged", sc.navigateToChanged)
 		}
 		if cmd.Flags().Changed("disableLiveReload") {
-			c.Set("disableLiveReload", s.disableLiveReload)
+			c.Set("disableLiveReload", sc.disableLiveReload)
 		}
 		if cmd.Flags().Changed("disableFastRender") {
-			c.Set("disableFastRender", s.disableFastRender)
+			c.Set("disableFastRender", sc.disableFastRender)
 		}
-		if s.serverWatch {
+		if cmd.Flags().Changed("disableBrowserError") {
+			c.Set("disableBrowserError", sc.disableBrowserError)
+		}
+		if sc.serverWatch {
 			c.Set("watch", true)
 		}
 
@@ -158,25 +167,25 @@ func (s *serverCmd) server(cmd *cobra.Command, args []string) error {
 			serverPorts = make([]int, 1)
 
 			if c.languages.IsMultihost() {
-				if !s.serverAppend {
+				if !sc.serverAppend {
 					err = newSystemError("--appendPort=false not supported when in multihost mode")
 				}
 				serverPorts = make([]int, len(c.languages))
 			}
 
-			currentServerPort := s.serverPort
+			currentServerPort := sc.serverPort
 
 			for i := 0; i < len(serverPorts); i++ {
-				l, err := net.Listen("tcp", net.JoinHostPort(s.serverInterface, strconv.Itoa(currentServerPort)))
+				l, err := net.Listen("tcp", net.JoinHostPort(sc.serverInterface, strconv.Itoa(currentServerPort)))
 				if err == nil {
 					l.Close()
 					serverPorts[i] = currentServerPort
 				} else {
-					if i == 0 && s.cmd.Flags().Changed("port") {
+					if i == 0 && sc.cmd.Flags().Changed("port") {
 						// port set explicitly by user -- he/she probably meant it!
 						err = newSystemErrorF("Server startup failed: %s", err)
 					}
-					jww.ERROR.Println("port", s.serverPort, "already in use, attempting to use an available port")
+					c.logger.FEEDBACK.Println("port", sc.serverPort, "already in use, attempting to use an available port")
 					sp, err := helpers.FindAvailablePort()
 					if err != nil {
 						err = newSystemError("Unable to find alternative port to use:", err)
@@ -190,9 +199,9 @@ func (s *serverCmd) server(cmd *cobra.Command, args []string) error {
 
 		c.serverPorts = serverPorts
 
-		c.Set("port", s.serverPort)
-		if s.liveReloadPort != -1 {
-			c.Set("liveReloadPort", s.liveReloadPort)
+		c.Set("port", sc.serverPort)
+		if sc.liveReloadPort != -1 {
+			c.Set("liveReloadPort", sc.liveReloadPort)
 		} else {
 			c.Set("liveReloadPort", serverPorts[0])
 		}
@@ -206,7 +215,7 @@ func (s *serverCmd) server(cmd *cobra.Command, args []string) error {
 				serverPort = serverPorts[0]
 			}
 
-			baseURL, err := s.fixURL(language, s.baseURL, serverPort)
+			baseURL, err := sc.fixURL(language, sc.baseURL, serverPort)
 			if err != nil {
 				return nil
 			}
@@ -223,10 +232,10 @@ func (s *serverCmd) server(cmd *cobra.Command, args []string) error {
 	}
 
 	if err := memStats(); err != nil {
-		jww.ERROR.Println("memstats error:", err)
+		jww.WARN.Println("memstats error:", err)
 	}
 
-	c, err := initializeConfig(true, &s.hugoBuilderCommon, s, cfgInit)
+	c, err := initializeConfig(true, true, &sc.hugoBuilderCommon, sc, cfgInit)
 	if err != nil {
 		return err
 	}
@@ -235,27 +244,23 @@ func (s *serverCmd) server(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	for _, s := range c.hugo.Sites {
+	for _, s := range c.hugo().Sites {
 		s.RegisterMediaTypes()
 	}
 
 	// Watch runs its own server as part of the routine
-	if s.serverWatch {
+	if sc.serverWatch {
 
 		watchDirs, err := c.getDirList()
 		if err != nil {
 			return err
 		}
 
-		baseWatchDir := c.Cfg.GetString("workingDir")
-		relWatchDirs := make([]string, len(watchDirs))
-		for i, dir := range watchDirs {
-			relWatchDirs[i], _ = helpers.GetRelativePath(dir, baseWatchDir)
+		watchGroups := helpers.ExtractAndGroupRootPaths(watchDirs)
+
+		for _, group := range watchGroups {
+			jww.FEEDBACK.Printf("Watching for changes in %s\n", group)
 		}
-
-		rootWatchDirs := strings.Join(helpers.UniqueStrings(helpers.ExtractRootPaths(relWatchDirs)), ",")
-
-		jww.FEEDBACK.Printf("Watching for changes in %s%s{%s}\n", baseWatchDir, helpers.FilePathSeparator, rootWatchDirs)
 		watcher, err := c.newWatcher(watchDirs...)
 
 		if err != nil {
@@ -266,15 +271,25 @@ func (s *serverCmd) server(cmd *cobra.Command, args []string) error {
 
 	}
 
-	return c.serve(s)
+	return c.serve(sc)
 
 }
 
+func getRootWatchDirsStr(baseDir string, watchDirs []string) string {
+	relWatchDirs := make([]string, len(watchDirs))
+	for i, dir := range watchDirs {
+		relWatchDirs[i], _ = helpers.GetRelativePath(dir, baseDir)
+	}
+
+	return strings.Join(helpers.UniqueStringsSorted(helpers.ExtractRootPaths(relWatchDirs)), ",")
+}
+
 type fileServer struct {
-	baseURLs []string
-	roots    []string
-	c        *commandeer
-	s        *serverCmd
+	baseURLs      []string
+	roots         []string
+	errorTemplate tpl.Template
+	c             *commandeer
+	s             *serverCmd
 }
 
 func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, error) {
@@ -288,7 +303,9 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, erro
 		publishDir = filepath.Join(publishDir, root)
 	}
 
-	absPublishDir := f.c.PathSpec().AbsPathify(publishDir)
+	absPublishDir := f.c.hugo().PathSpec.AbsPathify(publishDir)
+
+	jww.FEEDBACK.Printf("Environment: %q", f.c.hugo().Deps.Site.Hugo().Environment)
 
 	if i == 0 {
 		if f.s.renderToDisk {
@@ -298,33 +315,62 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, erro
 		}
 	}
 
-	httpFs := afero.NewHttpFs(f.c.Fs.Destination)
+	httpFs := afero.NewHttpFs(f.c.destinationFs)
 	fs := filesOnlyFs{httpFs.Dir(absPublishDir)}
 
-	doLiveReload := !f.s.buildWatch && !f.c.Cfg.GetBool("disableLiveReload")
-	fastRenderMode := doLiveReload && !f.c.Cfg.GetBool("disableFastRender")
-
-	if i == 0 && fastRenderMode {
+	if i == 0 && f.c.fastRenderMode {
 		jww.FEEDBACK.Println("Running in Fast Render Mode. For full rebuilds on change: hugo server --disableFastRender")
 	}
 
 	// We're only interested in the path
 	u, err := url.Parse(baseURL)
 	if err != nil {
-		return nil, "", "", fmt.Errorf("Invalid baseURL: %s", err)
+		return nil, "", "", errors.Wrap(err, "Invalid baseURL")
 	}
 
 	decorate := func(h http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if f.c.showErrorInBrowser {
+				// First check the error state
+				err := f.c.getErrorWithContext()
+				if err != nil {
+					w.WriteHeader(500)
+					var b bytes.Buffer
+					err := f.errorTemplate.Execute(&b, err)
+					if err != nil {
+						f.c.logger.ERROR.Println(err)
+					}
+					port = 1313
+					if !f.c.paused {
+						port = f.c.Cfg.GetInt("liveReloadPort")
+					}
+					fmt.Fprint(w, injectLiveReloadScript(&b, port))
+
+					return
+				}
+			}
+
 			if f.s.noHTTPCache {
 				w.Header().Set("Cache-Control", "no-store, no-cache, must-revalidate, max-age=0")
 				w.Header().Set("Pragma", "no-cache")
 			}
 
-			if fastRenderMode {
+			if f.c.fastRenderMode && f.c.buildErr == nil {
 				p := r.RequestURI
 				if strings.HasSuffix(p, "/") || strings.HasSuffix(p, "html") || strings.HasSuffix(p, "htm") {
+					if !f.c.visitedURLs.Contains(p) {
+						// If not already on stack, re-render that single page.
+						if err := f.c.partialReRender(p); err != nil {
+							f.c.handleBuildErr(err, fmt.Sprintf("Failed to render %q", p))
+							if f.c.showErrorInBrowser {
+								http.Redirect(w, r, p, http.StatusMovedPermanently)
+								return
+							}
+						}
+					}
+
 					f.c.visitedURLs.Add(p)
+
 				}
 			}
 			h.ServeHTTP(w, r)
@@ -345,9 +391,14 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, erro
 	return mu, u.String(), endpoint, nil
 }
 
+var logErrorRe = regexp.MustCompile(`(?s)ERROR \d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} `)
+
+func removeErrorPrefixFromLog(content string) string {
+	return logErrorRe.ReplaceAllLiteralString(content, "")
+}
 func (c *commandeer) serve(s *serverCmd) error {
 
-	isMultiHost := c.hugo.IsMultihost()
+	isMultiHost := c.hugo().IsMultihost()
 
 	var (
 		baseURLs []string
@@ -355,21 +406,27 @@ func (c *commandeer) serve(s *serverCmd) error {
 	)
 
 	if isMultiHost {
-		for _, s := range c.hugo.Sites {
+		for _, s := range c.hugo().Sites {
 			baseURLs = append(baseURLs, s.BaseURL.String())
-			roots = append(roots, s.Language.Lang)
+			roots = append(roots, s.Language().Lang)
 		}
 	} else {
-		s := c.hugo.Sites[0]
+		s := c.hugo().Sites[0]
 		baseURLs = []string{s.BaseURL.String()}
 		roots = []string{""}
 	}
 
+	templ, err := c.hugo().TextTmpl.Parse("__default_server_error", buildErrorTemplate)
+	if err != nil {
+		return err
+	}
+
 	srv := &fileServer{
-		baseURLs: baseURLs,
-		roots:    roots,
-		c:        c,
-		s:        s,
+		baseURLs:      baseURLs,
+		roots:         roots,
+		c:             c,
+		s:             s,
+		errorTemplate: templ,
 	}
 
 	doLiveReload := !c.Cfg.GetBool("disableLiveReload")
@@ -378,7 +435,7 @@ func (c *commandeer) serve(s *serverCmd) error {
 		livereload.Initialize()
 	}
 
-	var sigs = make(chan os.Signal)
+	var sigs = make(chan os.Signal, 1)
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
 
 	for i := range baseURLs {
@@ -392,7 +449,7 @@ func (c *commandeer) serve(s *serverCmd) error {
 		go func() {
 			err = http.ListenAndServe(endpoint, mu)
 			if err != nil {
-				jww.ERROR.Printf("Error: %s\n", err.Error())
+				c.logger.ERROR.Printf("Error: %s\n", err.Error())
 				os.Exit(1)
 			}
 		}()
@@ -453,7 +510,7 @@ func (sc *serverCmd) fixURL(cfg config.Provider, s string, port int) (string, er
 		if strings.Contains(u.Host, ":") {
 			u.Host, _, err = net.SplitHostPort(u.Host)
 			if err != nil {
-				return "", fmt.Errorf("Failed to split baseURL hostpost: %s", err)
+				return "", errors.Wrap(err, "Failed to split baseURL hostpost")
 			}
 		}
 		u.Host += fmt.Sprintf(":%d", port)

commands/server_errors.go

@@ -0,0 +1,95 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package commands
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/gohugoio/hugo/transform"
+	"github.com/gohugoio/hugo/transform/livereloadinject"
+)
+
+var buildErrorTemplate = `<!doctype html>
+<html class="no-js" lang="">
+	<head>
+		<meta charset="utf-8">
+		<title>Hugo Server: Error</title>
+		<style type="text/css">
+		body {
+			font-family: "Muli",avenir, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
+			font-size: 16px;
+			background-color: black;
+			color: rgba(255, 255, 255, 0.9);
+		}
+		main {
+			margin: auto;
+			width: 95%;
+			padding: 1rem;
+		}		
+		.version {
+			color: #ccc;
+			padding: 1rem 0;
+		}
+		.stack {
+			margin-top: 6rem;
+		}
+		pre {
+			white-space: pre-wrap;      
+			white-space: -moz-pre-wrap;  
+			white-space: -pre-wrap;     
+			white-space: -o-pre-wrap;    
+			word-wrap: break-word;     
+		}
+		.highlight {
+			overflow-x: auto;
+			padding: 0.75rem;
+			margin-bottom: 1rem;
+			background-color: #272822;
+			border: 1px solid black;
+		}
+		a {
+			color: #0594cb;
+			text-decoration: none;
+		}
+		a:hover {
+			color: #ccc;
+		}
+		</style>
+	</head>
+	<body>
+		<main>
+			{{ highlight .Error "apl" "noclasses=true,style=monokai" }}
+			{{ with .File }}
+			{{ $params := printf "noclasses=true,style=monokai,linenos=table,hl_lines=%d,linenostart=%d" (add .LinesPos 1) (sub .Position.LineNumber .LinesPos) }}
+			{{ $lexer := .ChromaLexer | default "go-html-template" }}
+			{{  highlight (delimit .Lines "\n") $lexer $params }}
+			{{ end }}
+			{{ with .StackTrace }}
+			{{ highlight . "apl" "noclasses=true,style=monokai" }}
+			{{ end }}
+			<p class="version">{{ .Version }}</p>
+			<a href="">Reload Page</a>
+		</main>
+</body>
+</html>
+`
+
+func injectLiveReloadScript(src io.Reader, port int) string {
+	var b bytes.Buffer
+	chain := transform.Chain{livereloadinject.New(port)}
+	chain.Apply(&b, src)
+
+	return b.String()
+}

commands/server_test.go

@@ -17,19 +17,25 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"runtime"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/gohugoio/hugo/helpers"
 
+	qt "github.com/frankban/quicktest"
 	"github.com/spf13/viper"
-	"github.com/stretchr/testify/require"
 )
 
 func TestServer(t *testing.T) {
-	assert := require.New(t)
-	dir, err := createSimpleTestSite(t)
-	assert.NoError(err)
+	if isWindowsCI() {
+		// TODO(bep) not sure why server tests have started to fail on the Windows CI server.
+		t.Skip("Skip server test on appveyor")
+	}
+	c := qt.New(t)
+	dir, err := createSimpleTestSite(t, testSiteConfig{})
+	c.Assert(err, qt.IsNil)
 
 	// Let us hope that this port is available on all systems ...
 	port := 1331
@@ -48,7 +54,7 @@ func TestServer(t *testing.T) {
 
 	go func() {
 		_, err = cmd.ExecuteC()
-		assert.NoError(err)
+		c.Assert(err, qt.IsNil)
 	}()
 
 	// There is no way to know exactly when the server is ready for connections.
@@ -57,11 +63,12 @@ func TestServer(t *testing.T) {
 	time.Sleep(2 * time.Second)
 
 	resp, err := http.Get("http://localhost:1331/")
-	assert.NoError(err)
+	c.Assert(err, qt.IsNil)
 	defer resp.Body.Close()
 	homeContent := helpers.ReaderToString(resp.Body)
 
-	assert.Contains(homeContent, "List: Hugo Commands")
+	c.Assert(homeContent, qt.Contains, "List: Hugo Commands")
+	c.Assert(homeContent, qt.Contains, "Environment: development")
 
 	// Stop the server.
 	stop <- true
@@ -90,20 +97,38 @@ func TestFixURL(t *testing.T) {
 		{"No config", "", "", true, 1313, "//localhost:1313/"},
 	}
 
-	for i, test := range tests {
-		b := newCommandsBuilder()
-		s := b.newServerCmd()
-		v := viper.New()
-		baseURL := test.CLIBaseURL
-		v.Set("baseURL", test.CfgBaseURL)
-		s.serverAppend = test.AppendPort
-		s.serverPort = test.Port
-		result, err := s.fixURL(v, baseURL, s.serverPort)
-		if err != nil {
-			t.Errorf("Test #%d %s: unexpected error %s", i, test.TestName, err)
-		}
-		if result != test.Result {
-			t.Errorf("Test #%d %s: expected %q, got %q", i, test.TestName, test.Result, result)
-		}
+	for _, test := range tests {
+		t.Run(test.TestName, func(t *testing.T) {
+			b := newCommandsBuilder()
+			s := b.newServerCmd()
+			v := viper.New()
+			baseURL := test.CLIBaseURL
+			v.Set("baseURL", test.CfgBaseURL)
+			s.serverAppend = test.AppendPort
+			s.serverPort = test.Port
+			result, err := s.fixURL(v, baseURL, s.serverPort)
+			if err != nil {
+				t.Errorf("Unexpected error %s", err)
+			}
+			if result != test.Result {
+				t.Errorf("Expected %q, got %q", test.Result, result)
+			}
+		})
 	}
 }
+
+func TestRemoveErrorPrefixFromLog(t *testing.T) {
+	c := qt.New(t)
+	content := `ERROR 2018/10/07 13:11:12 Error while rendering "home": template: _default/baseof.html:4:3: executing "main" at <partial "logo" .>: error calling partial: template: partials/logo.html:5:84: executing "partials/logo.html" at <$resized.AHeight>: can't evaluate field AHeight in type *resource.Image
+ERROR 2018/10/07 13:11:12 Rebuild failed: logged 1 error(s)
+`
+
+	withoutError := removeErrorPrefixFromLog(content)
+
+	c.Assert(strings.Contains(withoutError, "ERROR"), qt.Equals, false)
+
+}
+
+func isWindowsCI() bool {
+	return runtime.GOOS == "windows" && os.Getenv("CI") != ""
+}

commands/static_syncer.go

@@ -17,53 +17,44 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/gohugoio/hugo/hugolib/filesystems"
+
 	"github.com/fsnotify/fsnotify"
 	"github.com/gohugoio/hugo/helpers"
-	src "github.com/gohugoio/hugo/source"
 	"github.com/spf13/fsync"
 )
 
 type staticSyncer struct {
 	c *commandeer
-	d *src.Dirs
 }
 
 func newStaticSyncer(c *commandeer) (*staticSyncer, error) {
-	dirs, err := src.NewDirs(c.Fs, c.Cfg, c.DepsCfg.Logger)
-	if err != nil {
-		return nil, err
-	}
-
-	return &staticSyncer{c: c, d: dirs}, nil
+	return &staticSyncer{c: c}, nil
 }
 
-func (s *staticSyncer) isStatic(path string) bool {
-	return s.d.IsStatic(path)
+func (s *staticSyncer) isStatic(filename string) bool {
+	return s.c.hugo().BaseFs.SourceFilesystems.IsStatic(filename)
 }
 
 func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 	c := s.c
 
-	syncFn := func(dirs *src.Dirs, publishDir string) (uint64, error) {
-		staticSourceFs, err := dirs.CreateStaticFs()
-		if err != nil {
-			return 0, err
+	syncFn := func(sourceFs *filesystems.SourceFilesystem) (uint64, error) {
+		publishDir := c.hugo().PathSpec.PublishDir
+		// If root, remove the second '/'
+		if publishDir == "//" {
+			publishDir = helpers.FilePathSeparator
 		}
 
-		if dirs.Language != nil {
-			// Multihost setup
-			publishDir = filepath.Join(publishDir, dirs.Language.Lang)
-		}
-
-		if staticSourceFs == nil {
-			c.Logger.WARN.Println("No static directories found to sync")
-			return 0, nil
+		if sourceFs.PublishFolder != "" {
+			publishDir = filepath.Join(publishDir, sourceFs.PublishFolder)
 		}
 
 		syncer := fsync.NewSyncer()
 		syncer.NoTimes = c.Cfg.GetBool("noTimes")
 		syncer.NoChmod = c.Cfg.GetBool("noChmod")
-		syncer.SrcFs = staticSourceFs
+		syncer.ChmodFilter = chmodFilter
+		syncer.SrcFs = sourceFs.Fs
 		syncer.DestFs = c.Fs.Destination
 
 		// prevent spamming the log on changes
@@ -88,8 +79,8 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 
 			fromPath := ev.Name
 
-			// If we are here we already know the event took place in a static dir
-			relPath := dirs.MakeStaticPathRelative(fromPath)
+			relPath := sourceFs.MakePathRelative(fromPath)
+
 			if relPath == "" {
 				// Not member of this virtual host.
 				continue
@@ -105,7 +96,7 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 			// the source of that static file. In this case Hugo will incorrectly remove that file
 			// from the published directory.
 			if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove {
-				if _, err := staticSourceFs.Stat(relPath); os.IsNotExist(err) {
+				if _, err := sourceFs.Fs.Stat(relPath); os.IsNotExist(err) {
 					// If file doesn't exist in any static dir, remove it
 					toRemove := filepath.Join(publishDir, relPath)
 
@@ -116,10 +107,10 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 					logger.Println("Syncing", relPath, "to", publishDir)
 
 					if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil {
-						c.Logger.ERROR.Println(err)
+						c.logger.ERROR.Println(err)
 					}
 				} else {
-					c.Logger.ERROR.Println(err)
+					c.logger.ERROR.Println(err)
 				}
 
 				continue
@@ -128,7 +119,7 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 			// For all other event operations Hugo will sync static.
 			logger.Println("Syncing", relPath, "to", publishDir)
 			if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil {
-				c.Logger.ERROR.Println(err)
+				c.logger.ERROR.Println(err)
 			}
 		}
 

commands/version.go

@@ -14,11 +14,7 @@
 package commands
 
 import (
-	"runtime"
-	"strings"
-
-	"github.com/gohugoio/hugo/helpers"
-	"github.com/gohugoio/hugo/hugolib"
+	"github.com/gohugoio/hugo/common/hugo"
 	"github.com/spf13/cobra"
 	jww "github.com/spf13/jwalterweatherman"
 )
@@ -44,9 +40,5 @@ func newVersionCmd() *versionCmd {
 }
 
 func printHugoVersion() {
-	if hugolib.CommitHash == "" {
-		jww.FEEDBACK.Printf("Hugo Static Site Generator v%s %s/%s BuildDate: %s\n", helpers.CurrentHugoVersion, runtime.GOOS, runtime.GOARCH, hugolib.BuildDate)
-	} else {
-		jww.FEEDBACK.Printf("Hugo Static Site Generator v%s-%s %s/%s BuildDate: %s\n", helpers.CurrentHugoVersion, strings.ToUpper(hugolib.CommitHash), runtime.GOOS, runtime.GOARCH, hugolib.BuildDate)
-	}
+	jww.FEEDBACK.Println(hugo.BuildVersionString())
 }

common/collections/append.go

@@ -0,0 +1,112 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collections
+
+import (
+	"fmt"
+	"reflect"
+)
+
+// Append appends from to a slice to and returns the resulting slice.
+// If length of from is one and the only element is a slice of same type as to,
+// it will be appended.
+func Append(to interface{}, from ...interface{}) (interface{}, error) {
+	tov, toIsNil := indirect(reflect.ValueOf(to))
+
+	toIsNil = toIsNil || to == nil
+	var tot reflect.Type
+
+	if !toIsNil {
+		if tov.Kind() != reflect.Slice {
+			return nil, fmt.Errorf("expected a slice, got %T", to)
+		}
+
+		tot = tov.Type().Elem()
+		toIsNil = tov.Len() == 0
+
+		if len(from) == 1 {
+			fromv := reflect.ValueOf(from[0])
+			if fromv.Kind() == reflect.Slice {
+				if toIsNil {
+					// If we get nil []string, we just return the []string
+					return from[0], nil
+				}
+
+				fromt := reflect.TypeOf(from[0]).Elem()
+
+				// If we get []string []string, we append the from slice to to
+				if tot == fromt {
+					return reflect.AppendSlice(tov, fromv).Interface(), nil
+				} else if !fromt.AssignableTo(tot) {
+					// Fall back to a []interface{} slice.
+					return appendToInterfaceSliceFromValues(tov, fromv)
+
+				}
+			}
+		}
+	}
+
+	if toIsNil {
+		return Slice(from...), nil
+	}
+
+	for _, f := range from {
+		fv := reflect.ValueOf(f)
+		if !fv.Type().AssignableTo(tot) {
+			// Fall back to a []interface{} slice.
+			return appendToInterfaceSlice(tov, from...)
+		}
+		tov = reflect.Append(tov, fv)
+	}
+
+	return tov.Interface(), nil
+}
+
+func appendToInterfaceSliceFromValues(slice1, slice2 reflect.Value) ([]interface{}, error) {
+	var tos []interface{}
+
+	for _, slice := range []reflect.Value{slice1, slice2} {
+		for i := 0; i < slice.Len(); i++ {
+			tos = append(tos, slice.Index(i).Interface())
+		}
+	}
+
+	return tos, nil
+}
+
+func appendToInterfaceSlice(tov reflect.Value, from ...interface{}) ([]interface{}, error) {
+	var tos []interface{}
+
+	for i := 0; i < tov.Len(); i++ {
+		tos = append(tos, tov.Index(i).Interface())
+	}
+
+	tos = append(tos, from...)
+
+	return tos, nil
+}
+
+// indirect is borrowed from the Go stdlib: 'text/template/exec.go'
+// TODO(bep) consolidate
+func indirect(v reflect.Value) (rv reflect.Value, isNil bool) {
+	for ; v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface; v = v.Elem() {
+		if v.IsNil() {
+			return v, true
+		}
+		if v.Kind() == reflect.Interface && v.NumMethod() > 0 {
+			break
+		}
+	}
+	return v, false
+}

common/collections/append_test.go

@@ -0,0 +1,73 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collections
+
+import (
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestAppend(t *testing.T) {
+	t.Parallel()
+	c := qt.New(t)
+
+	for _, test := range []struct {
+		start    interface{}
+		addend   []interface{}
+		expected interface{}
+	}{
+		{[]string{"a", "b"}, []interface{}{"c"}, []string{"a", "b", "c"}},
+		{[]string{"a", "b"}, []interface{}{"c", "d", "e"}, []string{"a", "b", "c", "d", "e"}},
+		{[]string{"a", "b"}, []interface{}{[]string{"c", "d", "e"}}, []string{"a", "b", "c", "d", "e"}},
+		{nil, []interface{}{"a", "b"}, []string{"a", "b"}},
+		{nil, []interface{}{nil}, []interface{}{nil}},
+		{[]interface{}{}, []interface{}{[]string{"c", "d", "e"}}, []string{"c", "d", "e"}},
+		{tstSlicers{&tstSlicer{"a"}, &tstSlicer{"b"}},
+			[]interface{}{&tstSlicer{"c"}},
+			tstSlicers{&tstSlicer{"a"}, &tstSlicer{"b"}, &tstSlicer{"c"}}},
+		{&tstSlicers{&tstSlicer{"a"}, &tstSlicer{"b"}},
+			[]interface{}{&tstSlicer{"c"}},
+			tstSlicers{&tstSlicer{"a"},
+				&tstSlicer{"b"},
+				&tstSlicer{"c"}}},
+		{testSlicerInterfaces{&tstSlicerIn1{"a"}, &tstSlicerIn1{"b"}},
+			[]interface{}{&tstSlicerIn1{"c"}},
+			testSlicerInterfaces{&tstSlicerIn1{"a"}, &tstSlicerIn1{"b"}, &tstSlicerIn1{"c"}}},
+		//https://github.com/gohugoio/hugo/issues/5361
+		{[]string{"a", "b"}, []interface{}{tstSlicers{&tstSlicer{"a"}, &tstSlicer{"b"}}},
+			[]interface{}{"a", "b", &tstSlicer{"a"}, &tstSlicer{"b"}}},
+		{[]string{"a", "b"}, []interface{}{&tstSlicer{"a"}},
+			[]interface{}{"a", "b", &tstSlicer{"a"}}},
+		// Errors
+		{"", []interface{}{[]string{"a", "b"}}, false},
+		// No string concatenation.
+		{"ab",
+			[]interface{}{"c"},
+			false},
+	} {
+
+		result, err := Append(test.start, test.addend...)
+
+		if b, ok := test.expected.(bool); ok && !b {
+
+			c.Assert(err, qt.Not(qt.IsNil))
+			continue
+		}
+
+		c.Assert(err, qt.IsNil)
+		c.Assert(result, qt.DeepEquals, test.expected)
+	}
+
+}

common/collections/collections.go

@@ -0,0 +1,21 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package collections contains common Hugo functionality related to collection
+// handling.
+package collections
+
+// Grouper defines a very generic way to group items by a given key.
+type Grouper interface {
+	Group(key interface{}, items interface{}) (interface{}, error)
+}

common/collections/slice.go

@@ -0,0 +1,66 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collections
+
+import (
+	"reflect"
+)
+
+// Slicer defines a very generic way to create a typed slice. This is used
+// in collections.Slice template func to get types such as Pages, PageGroups etc.
+// instead of the less useful []interface{}.
+type Slicer interface {
+	Slice(items interface{}) (interface{}, error)
+}
+
+// Slice returns a slice of all passed arguments.
+func Slice(args ...interface{}) interface{} {
+	if len(args) == 0 {
+		return args
+	}
+
+	first := args[0]
+	firstType := reflect.TypeOf(first)
+
+	if firstType == nil {
+		return args
+	}
+
+	if g, ok := first.(Slicer); ok {
+		v, err := g.Slice(args)
+		if err == nil {
+			return v
+		}
+
+		// If Slice fails, the items are not of the same type and
+		// []interface{} is the best we can do.
+		return args
+	}
+
+	if len(args) > 1 {
+		// This can be a mix of types.
+		for i := 1; i < len(args); i++ {
+			if firstType != reflect.TypeOf(args[i]) {
+				// []interface{} is the best we can do
+				return args
+			}
+		}
+	}
+
+	slice := reflect.MakeSlice(reflect.SliceOf(firstType), len(args), len(args))
+	for i, arg := range args {
+		slice.Index(i).Set(reflect.ValueOf(arg))
+	}
+	return slice.Interface()
+}

common/collections/slice_test.go

@@ -0,0 +1,124 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collections
+
+import (
+	"errors"
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+var _ Slicer = (*tstSlicer)(nil)
+var _ Slicer = (*tstSlicerIn1)(nil)
+var _ Slicer = (*tstSlicerIn2)(nil)
+var _ testSlicerInterface = (*tstSlicerIn1)(nil)
+var _ testSlicerInterface = (*tstSlicerIn1)(nil)
+
+type testSlicerInterface interface {
+	Name() string
+}
+
+type testSlicerInterfaces []testSlicerInterface
+
+type tstSlicerIn1 struct {
+	TheName string
+}
+
+type tstSlicerIn2 struct {
+	TheName string
+}
+
+type tstSlicer struct {
+	TheName string
+}
+
+func (p *tstSlicerIn1) Slice(in interface{}) (interface{}, error) {
+	items := in.([]interface{})
+	result := make(testSlicerInterfaces, len(items))
+	for i, v := range items {
+		switch vv := v.(type) {
+		case testSlicerInterface:
+			result[i] = vv
+		default:
+			return nil, errors.New("invalid type")
+		}
+
+	}
+	return result, nil
+}
+
+func (p *tstSlicerIn2) Slice(in interface{}) (interface{}, error) {
+	items := in.([]interface{})
+	result := make(testSlicerInterfaces, len(items))
+	for i, v := range items {
+		switch vv := v.(type) {
+		case testSlicerInterface:
+			result[i] = vv
+		default:
+			return nil, errors.New("invalid type")
+		}
+	}
+	return result, nil
+}
+
+func (p *tstSlicerIn1) Name() string {
+	return p.TheName
+}
+
+func (p *tstSlicerIn2) Name() string {
+	return p.TheName
+}
+
+func (p *tstSlicer) Slice(in interface{}) (interface{}, error) {
+	items := in.([]interface{})
+	result := make(tstSlicers, len(items))
+	for i, v := range items {
+		switch vv := v.(type) {
+		case *tstSlicer:
+			result[i] = vv
+		default:
+			return nil, errors.New("invalid type")
+		}
+	}
+	return result, nil
+}
+
+type tstSlicers []*tstSlicer
+
+func TestSlice(t *testing.T) {
+	t.Parallel()
+	c := qt.New(t)
+
+	for i, test := range []struct {
+		args     []interface{}
+		expected interface{}
+	}{
+		{[]interface{}{"a", "b"}, []string{"a", "b"}},
+		{[]interface{}{&tstSlicer{"a"}, &tstSlicer{"b"}}, tstSlicers{&tstSlicer{"a"}, &tstSlicer{"b"}}},
+		{[]interface{}{&tstSlicer{"a"}, "b"}, []interface{}{&tstSlicer{"a"}, "b"}},
+		{[]interface{}{}, []interface{}{}},
+		{[]interface{}{nil}, []interface{}{nil}},
+		{[]interface{}{5, "b"}, []interface{}{5, "b"}},
+		{[]interface{}{&tstSlicerIn1{"a"}, &tstSlicerIn2{"b"}}, testSlicerInterfaces{&tstSlicerIn1{"a"}, &tstSlicerIn2{"b"}}},
+		{[]interface{}{&tstSlicerIn1{"a"}, &tstSlicer{"b"}}, []interface{}{&tstSlicerIn1{"a"}, &tstSlicer{"b"}}},
+	} {
+		errMsg := qt.Commentf("[%d] %v", i, test.args)
+
+		result := Slice(test.args...)
+
+		c.Assert(test.expected, qt.DeepEquals, result, errMsg)
+	}
+
+}

common/herrors/error_locator.go

@@ -0,0 +1,255 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package errors contains common Hugo errors and error related utilities.
+package herrors
+
+import (
+	"io"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+
+	"github.com/gohugoio/hugo/common/text"
+
+	"github.com/spf13/afero"
+)
+
+// LineMatcher contains the elements used to match an error to a line
+type LineMatcher struct {
+	Position text.Position
+	Error    error
+
+	LineNumber int
+	Offset     int
+	Line       string
+}
+
+// LineMatcherFn is used to match a line with an error.
+type LineMatcherFn func(m LineMatcher) bool
+
+// SimpleLineMatcher simply matches by line number.
+var SimpleLineMatcher = func(m LineMatcher) bool {
+	return m.Position.LineNumber == m.LineNumber
+}
+
+var _ text.Positioner = ErrorContext{}
+
+// ErrorContext contains contextual information about an error. This will
+// typically be the lines surrounding some problem in a file.
+type ErrorContext struct {
+
+	// If a match will contain the matched line and up to 2 lines before and after.
+	// Will be empty if no match.
+	Lines []string
+
+	// The position of the error in the Lines above. 0 based.
+	LinesPos int
+
+	position text.Position
+
+	// The lexer to use for syntax highlighting.
+	// https://gohugo.io/content-management/syntax-highlighting/#list-of-chroma-highlighting-languages
+	ChromaLexer string
+}
+
+// Position returns the text position of this error.
+func (e ErrorContext) Position() text.Position {
+	return e.position
+}
+
+var _ causer = (*ErrorWithFileContext)(nil)
+
+// ErrorWithFileContext is an error with some additional file context related
+// to that error.
+type ErrorWithFileContext struct {
+	cause error
+	ErrorContext
+}
+
+func (e *ErrorWithFileContext) Error() string {
+	pos := e.Position()
+	if pos.IsValid() {
+		return pos.String() + ": " + e.cause.Error()
+	}
+	return e.cause.Error()
+}
+
+func (e *ErrorWithFileContext) Cause() error {
+	return e.cause
+}
+
+// WithFileContextForFile will try to add a file context with lines matching the given matcher.
+// If no match could be found, the original error is returned with false as the second return value.
+func WithFileContextForFile(e error, realFilename, filename string, fs afero.Fs, matcher LineMatcherFn) (error, bool) {
+	f, err := fs.Open(filename)
+	if err != nil {
+		return e, false
+	}
+	defer f.Close()
+	return WithFileContext(e, realFilename, f, matcher)
+}
+
+// WithFileContextForFile will try to add a file context with lines matching the given matcher.
+// If no match could be found, the original error is returned with false as the second return value.
+func WithFileContext(e error, realFilename string, r io.Reader, matcher LineMatcherFn) (error, bool) {
+	if e == nil {
+		panic("error missing")
+	}
+	le := UnwrapFileError(e)
+
+	if le == nil {
+		var ok bool
+		if le, ok = ToFileError("", e).(FileError); !ok {
+			return e, false
+		}
+	}
+
+	var errCtx ErrorContext
+
+	posle := le.Position()
+
+	if posle.Offset != -1 {
+		errCtx = locateError(r, le, func(m LineMatcher) bool {
+			if posle.Offset >= m.Offset && posle.Offset < m.Offset+len(m.Line) {
+				lno := posle.LineNumber - m.Position.LineNumber + m.LineNumber
+				m.Position = text.Position{LineNumber: lno}
+			}
+			return matcher(m)
+		})
+	} else {
+		errCtx = locateError(r, le, matcher)
+	}
+
+	pos := &errCtx.position
+
+	if pos.LineNumber == -1 {
+		return e, false
+	}
+
+	pos.Filename = realFilename
+
+	if le.Type() != "" {
+		errCtx.ChromaLexer = chromaLexerFromType(le.Type())
+	} else {
+		errCtx.ChromaLexer = chromaLexerFromFilename(realFilename)
+	}
+
+	return &ErrorWithFileContext{cause: e, ErrorContext: errCtx}, true
+}
+
+// UnwrapErrorWithFileContext tries to unwrap an ErrorWithFileContext from err.
+// It returns nil if this is not possible.
+func UnwrapErrorWithFileContext(err error) *ErrorWithFileContext {
+	for err != nil {
+		switch v := err.(type) {
+		case *ErrorWithFileContext:
+			return v
+		case causer:
+			err = v.Cause()
+		default:
+			return nil
+		}
+	}
+	return nil
+}
+
+func chromaLexerFromType(fileType string) string {
+	switch fileType {
+	case "html", "htm":
+		return "go-html-template"
+	}
+	return fileType
+}
+
+func extNoDelimiter(filename string) string {
+	return strings.TrimPrefix(filepath.Ext(filename), ".")
+}
+
+func chromaLexerFromFilename(filename string) string {
+	if strings.Contains(filename, "layouts") {
+		return "go-html-template"
+	}
+
+	ext := extNoDelimiter(filename)
+	return chromaLexerFromType(ext)
+}
+
+func locateErrorInString(src string, matcher LineMatcherFn) ErrorContext {
+	return locateError(strings.NewReader(src), &fileError{}, matcher)
+}
+
+func locateError(r io.Reader, le FileError, matches LineMatcherFn) ErrorContext {
+	if le == nil {
+		panic("must provide an error")
+	}
+
+	errCtx := ErrorContext{position: text.Position{LineNumber: -1, ColumnNumber: 1, Offset: -1}, LinesPos: -1}
+
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		return errCtx
+	}
+
+	pos := &errCtx.position
+	lepos := le.Position()
+
+	lines := strings.Split(string(b), "\n")
+
+	if lepos.ColumnNumber >= 0 {
+		pos.ColumnNumber = lepos.ColumnNumber
+	}
+
+	lineNo := 0
+	posBytes := 0
+
+	for li, line := range lines {
+		lineNo = li + 1
+		m := LineMatcher{
+			Position:   le.Position(),
+			Error:      le,
+			LineNumber: lineNo,
+			Offset:     posBytes,
+			Line:       line,
+		}
+		if errCtx.LinesPos == -1 && matches(m) {
+			pos.LineNumber = lineNo
+			break
+		}
+
+		posBytes += len(line)
+	}
+
+	if pos.LineNumber != -1 {
+		low := pos.LineNumber - 3
+		if low < 0 {
+			low = 0
+		}
+
+		if pos.LineNumber > 2 {
+			errCtx.LinesPos = 2
+		} else {
+			errCtx.LinesPos = pos.LineNumber - 1
+		}
+
+		high := pos.LineNumber + 2
+		if high > len(lines) {
+			high = len(lines)
+		}
+
+		errCtx.Lines = lines[low:high]
+
+	}
+
+	return errCtx
+}

common/herrors/error_locator_test.go

@@ -0,0 +1,129 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package errors contains common Hugo errors and error related utilities.
+package herrors
+
+import (
+	"strings"
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestErrorLocator(t *testing.T) {
+	c := qt.New(t)
+
+	lineMatcher := func(m LineMatcher) bool {
+		return strings.Contains(m.Line, "THEONE")
+	}
+
+	lines := `LINE 1
+LINE 2
+LINE 3
+LINE 4
+This is THEONE
+LINE 6
+LINE 7
+LINE 8
+`
+
+	location := locateErrorInString(lines, lineMatcher)
+	c.Assert(location.Lines, qt.DeepEquals, []string{"LINE 3", "LINE 4", "This is THEONE", "LINE 6", "LINE 7"})
+
+	pos := location.Position()
+	c.Assert(pos.LineNumber, qt.Equals, 5)
+	c.Assert(location.LinesPos, qt.Equals, 2)
+
+	c.Assert(locateErrorInString(`This is THEONE`, lineMatcher).Lines, qt.DeepEquals, []string{"This is THEONE"})
+
+	location = locateErrorInString(`L1
+This is THEONE
+L2
+`, lineMatcher)
+	c.Assert(location.Position().LineNumber, qt.Equals, 2)
+	c.Assert(location.LinesPos, qt.Equals, 1)
+	c.Assert(location.Lines, qt.DeepEquals, []string{"L1", "This is THEONE", "L2", ""})
+
+	location = locateErrorInString(`This is THEONE
+L2
+`, lineMatcher)
+	c.Assert(location.LinesPos, qt.Equals, 0)
+	c.Assert(location.Lines, qt.DeepEquals, []string{"This is THEONE", "L2", ""})
+
+	location = locateErrorInString(`L1
+This THEONE
+`, lineMatcher)
+	c.Assert(location.Lines, qt.DeepEquals, []string{"L1", "This THEONE", ""})
+	c.Assert(location.LinesPos, qt.Equals, 1)
+
+	location = locateErrorInString(`L1
+L2
+This THEONE
+`, lineMatcher)
+	c.Assert(location.Lines, qt.DeepEquals, []string{"L1", "L2", "This THEONE", ""})
+	c.Assert(location.LinesPos, qt.Equals, 2)
+
+	location = locateErrorInString("NO MATCH", lineMatcher)
+	c.Assert(location.Position().LineNumber, qt.Equals, -1)
+	c.Assert(location.LinesPos, qt.Equals, -1)
+	c.Assert(len(location.Lines), qt.Equals, 0)
+
+	lineMatcher = func(m LineMatcher) bool {
+		return m.LineNumber == 6
+	}
+
+	location = locateErrorInString(`A
+B
+C
+D
+E
+F
+G
+H
+I
+J`, lineMatcher)
+
+	c.Assert(location.Lines, qt.DeepEquals, []string{"D", "E", "F", "G", "H"})
+	c.Assert(location.Position().LineNumber, qt.Equals, 6)
+	c.Assert(location.LinesPos, qt.Equals, 2)
+
+	// Test match EOF
+	lineMatcher = func(m LineMatcher) bool {
+		return m.LineNumber == 4
+	}
+
+	location = locateErrorInString(`A
+B
+C
+`, lineMatcher)
+
+	c.Assert(location.Lines, qt.DeepEquals, []string{"B", "C", ""})
+	c.Assert(location.Position().LineNumber, qt.Equals, 4)
+	c.Assert(location.LinesPos, qt.Equals, 2)
+
+	offsetMatcher := func(m LineMatcher) bool {
+		return m.Offset == 1
+	}
+
+	location = locateErrorInString(`A
+B
+C
+D
+E`, offsetMatcher)
+
+	c.Assert(location.Lines, qt.DeepEquals, []string{"A", "B", "C", "D"})
+	c.Assert(location.Position().LineNumber, qt.Equals, 2)
+	c.Assert(location.LinesPos, qt.Equals, 1)
+
+}

common/herrors/errors.go

@@ -0,0 +1,65 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package herrors contains common Hugo errors and error related utilities.
+package herrors
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"runtime/debug"
+
+	_errors "github.com/pkg/errors"
+)
+
+// As defined in https://godoc.org/github.com/pkg/errors
+type causer interface {
+	Cause() error
+}
+
+type stackTracer interface {
+	StackTrace() _errors.StackTrace
+}
+
+// PrintStackTrace prints the error's stack trace to stdoud.
+func PrintStackTrace(err error) {
+	FprintStackTrace(os.Stdout, err)
+}
+
+// FprintStackTrace prints the error's stack trace to w.
+func FprintStackTrace(w io.Writer, err error) {
+	if err, ok := err.(stackTracer); ok {
+		for _, f := range err.StackTrace() {
+			fmt.Fprintf(w, "%+s:%d\n", f, f)
+		}
+	}
+}
+
+// Recover is a helper function that can be used to capture panics.
+// Put this at the top of a method/function that crashes in a template:
+//     defer herrors.Recover()
+func Recover(args ...interface{}) {
+	if r := recover(); r != nil {
+		args = append(args, "stacktrace from panic: \n"+string(debug.Stack()), "\n")
+		fmt.Println(args...)
+	}
+
+}
+
+// ErrFeatureNotAvailable denotes that a feature is unavailable.
+//
+// We will, at least to begin with, make some Hugo features (SCSS with libsass) optional,
+// and this error is used to signal those situations.
+var ErrFeatureNotAvailable = errors.New("this feature is not available in your current Hugo version, see https://goo.gl/YMrWcn for more information")

common/herrors/file_error.go

@@ -0,0 +1,133 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitatio	ns under the License.
+
+package herrors
+
+import (
+	"encoding/json"
+
+	"github.com/gohugoio/hugo/common/text"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	_ causer = (*fileError)(nil)
+)
+
+// FileError represents an error when handling a file: Parsing a config file,
+// execute a template etc.
+type FileError interface {
+	error
+
+	text.Positioner
+
+	// A string identifying the type of file, e.g. JSON, TOML, markdown etc.
+	Type() string
+}
+
+var _ FileError = (*fileError)(nil)
+
+type fileError struct {
+	position text.Position
+
+	fileType string
+
+	cause error
+}
+
+// Position returns the text position of this error.
+func (e fileError) Position() text.Position {
+	return e.position
+}
+
+func (e *fileError) Type() string {
+	return e.fileType
+}
+
+func (e *fileError) Error() string {
+	if e.cause == nil {
+		return ""
+	}
+	return e.cause.Error()
+}
+
+func (f *fileError) Cause() error {
+	return f.cause
+}
+
+// NewFileError creates a new FileError.
+func NewFileError(fileType string, offset, lineNumber, columnNumber int, err error) FileError {
+	pos := text.Position{Offset: offset, LineNumber: lineNumber, ColumnNumber: columnNumber}
+	return &fileError{cause: err, fileType: fileType, position: pos}
+}
+
+// UnwrapFileError tries to unwrap a FileError from err.
+// It returns nil if this is not possible.
+func UnwrapFileError(err error) FileError {
+	for err != nil {
+		switch v := err.(type) {
+		case FileError:
+			return v
+		case causer:
+			err = v.Cause()
+		default:
+			return nil
+		}
+	}
+	return nil
+}
+
+// ToFileErrorWithOffset will return a new FileError with a line number
+// with the given offset from the original.
+func ToFileErrorWithOffset(fe FileError, offset int) FileError {
+	pos := fe.Position()
+	return ToFileErrorWithLineNumber(fe, pos.LineNumber+offset)
+}
+
+// ToFileErrorWithOffset will return a new FileError with the given line number.
+func ToFileErrorWithLineNumber(fe FileError, lineNumber int) FileError {
+	pos := fe.Position()
+	pos.LineNumber = lineNumber
+	return &fileError{cause: fe, fileType: fe.Type(), position: pos}
+}
+
+// ToFileError will convert the given error to an error supporting
+// the FileError interface.
+func ToFileError(fileType string, err error) FileError {
+	for _, handle := range lineNumberExtractors {
+		lno, col := handle(err)
+		offset, typ := extractOffsetAndType(err)
+		if fileType == "" {
+			fileType = typ
+		}
+
+		if lno > 0 || offset != -1 {
+			return NewFileError(fileType, offset, lno, col, err)
+		}
+	}
+	// Fall back to the pointing to line number 1.
+	return NewFileError(fileType, -1, 1, 1, err)
+}
+
+func extractOffsetAndType(e error) (int, string) {
+	e = errors.Cause(e)
+	switch v := e.(type) {
+	case *json.UnmarshalTypeError:
+		return int(v.Offset), "json"
+	case *json.SyntaxError:
+		return int(v.Offset), "json"
+	default:
+		return -1, ""
+	}
+}

common/herrors/file_error_test.go

@@ -0,0 +1,56 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package herrors
+
+import (
+	"testing"
+
+	"github.com/pkg/errors"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestToLineNumberError(t *testing.T) {
+	t.Parallel()
+
+	c := qt.New(t)
+
+	for i, test := range []struct {
+		in           error
+		offset       int
+		lineNumber   int
+		columnNumber int
+	}{
+		{errors.New("no line number for you"), 0, 1, 1},
+		{errors.New(`template: _default/single.html:4:15: executing "_default/single.html" at <.Titles>: can't evaluate field Titles in type *hugolib.PageOutput`), 0, 4, 15},
+		{errors.New("parse failed: template: _default/bundle-resource-meta.html:11: unexpected in operand"), 0, 11, 1},
+		{errors.New(`failed:: template: _default/bundle-resource-meta.html:2:7: executing "main" at <.Titles>`), 0, 2, 7},
+		{errors.New("error in front matter: Near line 32 (last key parsed 'title')"), 0, 32, 1},
+		{errors.New(`failed to load translations: (6, 7): was expecting token =, but got "g" instead`), 0, 6, 7},
+	} {
+
+		got := ToFileError("template", test.in)
+
+		errMsg := qt.Commentf("[%d][%T]", i, got)
+		le, ok := got.(FileError)
+		c.Assert(ok, qt.Equals, true)
+
+		c.Assert(ok, qt.Equals, true, errMsg)
+		pos := le.Position()
+		c.Assert(pos.LineNumber, qt.Equals, test.lineNumber, errMsg)
+		c.Assert(pos.ColumnNumber, qt.Equals, test.columnNumber, errMsg)
+		c.Assert(errors.Cause(got), qt.Not(qt.IsNil))
+	}
+
+}

common/herrors/line_number_extractors.go

@@ -0,0 +1,66 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitatio	ns under the License.
+
+package herrors
+
+import (
+	"regexp"
+	"strconv"
+)
+
+var lineNumberExtractors = []lineNumberExtractor{
+	// Template/shortcode parse errors
+	newLineNumberErrHandlerFromRegexp(".*:(\\d+):(\\d*):"),
+	newLineNumberErrHandlerFromRegexp(".*:(\\d+):"),
+
+	// TOML parse errors
+	newLineNumberErrHandlerFromRegexp(".*Near line (\\d+)(\\s.*)"),
+
+	// YAML parse errors
+	newLineNumberErrHandlerFromRegexp("line (\\d+):"),
+
+	// i18n bundle errors
+	newLineNumberErrHandlerFromRegexp("\\((\\d+),\\s(\\d*)"),
+}
+
+type lineNumberExtractor func(e error) (int, int)
+
+func newLineNumberErrHandlerFromRegexp(expression string) lineNumberExtractor {
+	re := regexp.MustCompile(expression)
+	return extractLineNo(re)
+}
+
+func extractLineNo(re *regexp.Regexp) lineNumberExtractor {
+	return func(e error) (int, int) {
+		if e == nil {
+			panic("no error")
+		}
+		col := 1
+		s := e.Error()
+		m := re.FindStringSubmatch(s)
+		if len(m) >= 2 {
+			lno, _ := strconv.Atoi(m[1])
+			if len(m) > 2 {
+				col, _ = strconv.Atoi(m[2])
+			}
+
+			if col <= 0 {
+				col = 1
+			}
+
+			return lno, col
+		}
+
+		return -1, col
+	}
+}

common/hreflect/helpers.go

@@ -0,0 +1,91 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+// Some functions in this file (see comments) is based on the Go source code,
+// copyright The Go Authors and  governed by a BSD-style license.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package hreflect contains reflect helpers.
+package hreflect
+
+import (
+	"reflect"
+
+	"github.com/gohugoio/hugo/common/types"
+)
+
+// IsTruthful returns whether in represents a truthful value.
+// See IsTruthfulValue
+func IsTruthful(in interface{}) bool {
+	switch v := in.(type) {
+	case reflect.Value:
+		return IsTruthfulValue(v)
+	default:
+		return IsTruthfulValue(reflect.ValueOf(in))
+	}
+
+}
+
+var zeroType = reflect.TypeOf((*types.Zeroer)(nil)).Elem()
+
+// IsTruthfulValue returns whether the given value has a meaningful truth value.
+// This is based on template.IsTrue in Go's stdlib, but also considers
+// IsZero and any interface value will be unwrapped before it's considered
+// for truthfulness.
+//
+// Based on:
+// https://github.com/golang/go/blob/178a2c42254166cffed1b25fb1d3c7a5727cada6/src/text/template/exec.go#L306
+func IsTruthfulValue(val reflect.Value) (truth bool) {
+	val = indirectInterface(val)
+
+	if !val.IsValid() {
+		// Something like var x interface{}, never set. It's a form of nil.
+		return
+	}
+
+	if val.Type().Implements(zeroType) {
+		return !val.Interface().(types.Zeroer).IsZero()
+	}
+
+	switch val.Kind() {
+	case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
+		truth = val.Len() > 0
+	case reflect.Bool:
+		truth = val.Bool()
+	case reflect.Complex64, reflect.Complex128:
+		truth = val.Complex() != 0
+	case reflect.Chan, reflect.Func, reflect.Ptr, reflect.Interface:
+		truth = !val.IsNil()
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		truth = val.Int() != 0
+	case reflect.Float32, reflect.Float64:
+		truth = val.Float() != 0
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		truth = val.Uint() != 0
+	case reflect.Struct:
+		truth = true // Struct values are always true.
+	default:
+		return
+	}
+
+	return
+}
+
+// Based on: https://github.com/golang/go/blob/178a2c42254166cffed1b25fb1d3c7a5727cada6/src/text/template/exec.go#L931
+func indirectInterface(v reflect.Value) reflect.Value {
+	if v.Kind() != reflect.Interface {
+		return v
+	}
+	if v.IsNil() {
+		return reflect.Value{}
+	}
+	return v.Elem()
+}

common/hreflect/helpers_test.go

@@ -0,0 +1,42 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hreflect
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestIsTruthful(t *testing.T) {
+	c := qt.New(t)
+
+	c.Assert(IsTruthful(true), qt.Equals, true)
+	c.Assert(IsTruthful(false), qt.Equals, false)
+	c.Assert(IsTruthful(time.Now()), qt.Equals, true)
+	c.Assert(IsTruthful(time.Time{}), qt.Equals, false)
+}
+
+func BenchmarkIsTruthFul(b *testing.B) {
+	v := reflect.ValueOf("Hugo")
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if !IsTruthfulValue(v) {
+			b.Fatal("not truthful")
+		}
+	}
+}

common/hugio/copy.go

@@ -0,0 +1,90 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugio
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+
+	"github.com/spf13/afero"
+)
+
+// CopyFile copies a file.
+func CopyFile(fs afero.Fs, from, to string) error {
+	sf, err := os.Open(from)
+	if err != nil {
+		return err
+	}
+	defer sf.Close()
+	df, err := os.Create(to)
+	if err != nil {
+		return err
+	}
+	defer df.Close()
+	_, err = io.Copy(df, sf)
+	if err == nil {
+		si, err := os.Stat(from)
+		if err != nil {
+			err = os.Chmod(to, si.Mode())
+
+			if err != nil {
+				return err
+			}
+		}
+
+	}
+	return nil
+}
+
+// CopyDir copies a directory.
+func CopyDir(fs afero.Fs, from, to string, shouldCopy func(filename string) bool) error {
+	fi, err := os.Stat(from)
+	if err != nil {
+		return err
+	}
+
+	if !fi.IsDir() {
+		return errors.Errorf("%q is not a directory", from)
+	}
+
+	err = fs.MkdirAll(to, 0777) // before umask
+	if err != nil {
+		return err
+	}
+
+	entries, _ := ioutil.ReadDir(from)
+	for _, entry := range entries {
+		fromFilename := filepath.Join(from, entry.Name())
+		toFilename := filepath.Join(to, entry.Name())
+		if entry.IsDir() {
+			if shouldCopy != nil && !shouldCopy(fromFilename) {
+				continue
+			}
+			if err := CopyDir(fs, fromFilename, toFilename, shouldCopy); err != nil {
+				return err
+			}
+		} else {
+			if err := CopyFile(fs, fromFilename, toFilename); err != nil {
+				return err
+			}
+		}
+
+	}
+
+	return nil
+}

common/hugio/readers.go

@@ -0,0 +1,54 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugio
+
+import (
+	"io"
+	"strings"
+)
+
+// ReadSeeker wraps io.Reader and io.Seeker.
+type ReadSeeker interface {
+	io.Reader
+	io.Seeker
+}
+
+// ReadSeekCloser is implemented by afero.File. We use this as the common type for
+// content in Resource objects, even for strings.
+type ReadSeekCloser interface {
+	ReadSeeker
+	io.Closer
+}
+
+// ReadSeekerNoOpCloser implements ReadSeekCloser by doing nothing in Close.
+// TODO(bep) rename this and simila to ReadSeekerNopCloser, naming used in stdlib, which kind of makes sense.
+type ReadSeekerNoOpCloser struct {
+	ReadSeeker
+}
+
+// Close does nothing.
+func (r ReadSeekerNoOpCloser) Close() error {
+	return nil
+}
+
+// NewReadSeekerNoOpCloser creates a new ReadSeekerNoOpCloser with the given ReadSeeker.
+func NewReadSeekerNoOpCloser(r ReadSeeker) ReadSeekerNoOpCloser {
+	return ReadSeekerNoOpCloser{r}
+}
+
+// NewReadSeekerNoOpCloserFromString uses strings.NewReader to create a new ReadSeekerNoOpCloser
+// from the given string.
+func NewReadSeekerNoOpCloserFromString(content string) ReadSeekerNoOpCloser {
+	return ReadSeekerNoOpCloser{strings.NewReader(content)}
+}

common/hugio/writers.go

@@ -0,0 +1,76 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugio
+
+import (
+	"io"
+	"io/ioutil"
+)
+
+type multiWriteCloser struct {
+	io.Writer
+	closers []io.WriteCloser
+}
+
+func (m multiWriteCloser) Close() error {
+	var err error
+	for _, c := range m.closers {
+		if closeErr := c.Close(); err != nil {
+			err = closeErr
+		}
+	}
+	return err
+}
+
+// NewMultiWriteCloser creates a new io.WriteCloser that duplicates its writes to all the
+// provided writers.
+func NewMultiWriteCloser(writeClosers ...io.WriteCloser) io.WriteCloser {
+	writers := make([]io.Writer, len(writeClosers))
+	for i, w := range writeClosers {
+		writers[i] = w
+	}
+	return multiWriteCloser{Writer: io.MultiWriter(writers...), closers: writeClosers}
+}
+
+// ToWriteCloser creates an io.WriteCloser from the given io.Writer.
+// If it's not already, one will be created with a Close method that does nothing.
+func ToWriteCloser(w io.Writer) io.WriteCloser {
+	if rw, ok := w.(io.WriteCloser); ok {
+		return rw
+	}
+
+	return struct {
+		io.Writer
+		io.Closer
+	}{
+		w,
+		ioutil.NopCloser(nil),
+	}
+}
+
+// ToReadCloser creates an io.ReadCloser from the given io.Reader.
+// If it's not already, one will be created with a Close method that does nothing.
+func ToReadCloser(r io.Reader) io.ReadCloser {
+	if rc, ok := r.(io.ReadCloser); ok {
+		return rc
+	}
+
+	return struct {
+		io.Reader
+		io.Closer
+	}{
+		r,
+		ioutil.NopCloser(nil),
+	}
+}

common/hugo/hugo.go

@@ -0,0 +1,67 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugo
+
+import (
+	"fmt"
+	"html/template"
+)
+
+const (
+	EnvironmentDevelopment = "development"
+	EnvironmentProduction  = "production"
+)
+
+var (
+	// commitHash contains the current Git revision. Use make to build to make
+	// sure this gets set.
+	commitHash string
+
+	// buildDate contains the date of the current build.
+	buildDate string
+)
+
+// Info contains information about the current Hugo environment
+type Info struct {
+	CommitHash string
+	BuildDate  string
+
+	// The build environment.
+	// Defaults are "production" (hugo) and "development" (hugo server).
+	// This can also be set by the user.
+	// It can be any string, but it will be all lower case.
+	Environment string
+}
+
+// Version returns the current version as a comparable version string.
+func (i Info) Version() VersionString {
+	return CurrentVersion.Version()
+}
+
+// Generator a Hugo meta generator HTML tag.
+func (i Info) Generator() template.HTML {
+	return template.HTML(fmt.Sprintf(`<meta name="generator" content="Hugo %s" />`, CurrentVersion.String()))
+}
+
+// NewInfo creates a new Hugo Info object.
+func NewInfo(environment string) Info {
+	if environment == "" {
+		environment = EnvironmentProduction
+	}
+	return Info{
+		CommitHash:  commitHash,
+		BuildDate:   buildDate,
+		Environment: environment,
+	}
+}

common/hugo/hugo_test.go

@@ -0,0 +1,35 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugo
+
+import (
+	"fmt"
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestHugoInfo(t *testing.T) {
+	c := qt.New(t)
+
+	hugoInfo := NewInfo("")
+
+	c.Assert(hugoInfo.Version(), qt.Equals, CurrentVersion.Version())
+	c.Assert(fmt.Sprintf("%T", VersionString("")), qt.Equals, fmt.Sprintf("%T", hugoInfo.Version()))
+	c.Assert(hugoInfo.CommitHash, qt.Equals, commitHash)
+	c.Assert(hugoInfo.BuildDate, qt.Equals, buildDate)
+	c.Assert(hugoInfo.Environment, qt.Equals, "production")
+	c.Assert(string(hugoInfo.Generator()), qt.Contains, fmt.Sprintf("Hugo %s", hugoInfo.Version()))
+
+}

common/hugo/vars_extended.go

@@ -1,4 +1,4 @@
-// Copyright 2017-present The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,13 +11,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package hugolib
+// +build extended
 
-import (
-	"github.com/gohugoio/hugo/resource"
-)
+package hugo
 
-var (
-	_ resource.Resource = (*Page)(nil)
-	_ resource.Resource = (*PageOutput)(nil)
-)
+var IsExtended = true

common/hugo/vars_regular.go

@@ -0,0 +1,18 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !extended
+
+package hugo
+
+var IsExtended = false

common/hugo/version.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,18 +11,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package helpers
+package hugo
 
 import (
 	"fmt"
+	"strconv"
+
+	"runtime"
 	"strings"
 
 	"github.com/gohugoio/hugo/compare"
 	"github.com/spf13/cast"
 )
 
-// HugoVersion represents the Hugo build version.
-type HugoVersion struct {
+// Version represents the Hugo build version.
+type Version struct {
 	// Major and minor version.
 	Number float32
 
@@ -35,32 +38,34 @@ type HugoVersion struct {
 }
 
 var (
-	_ compare.Eqer     = (*HugoVersionString)(nil)
-	_ compare.Comparer = (*HugoVersionString)(nil)
+	_ compare.Eqer     = (*VersionString)(nil)
+	_ compare.Comparer = (*VersionString)(nil)
 )
 
-type HugoVersionString string
-
-func (v HugoVersion) String() string {
-	return hugoVersion(v.Number, v.PatchLevel, v.Suffix)
+func (v Version) String() string {
+	return version(v.Number, v.PatchLevel, v.Suffix)
 }
 
-func (v HugoVersion) Version() HugoVersionString {
-	return HugoVersionString(v.String())
+// Version returns the Hugo version.
+func (v Version) Version() VersionString {
+	return VersionString(v.String())
 }
 
-func (h HugoVersionString) String() string {
+// VersionString represents a Hugo version string.
+type VersionString string
+
+func (h VersionString) String() string {
 	return string(h)
 }
 
-// Implements compare.Comparer
-func (h HugoVersionString) Compare(other interface{}) int {
-	v := MustParseHugoVersion(h.String())
+// Compare implements the compare.Comparer interface.
+func (h VersionString) Compare(other interface{}) int {
+	v := MustParseVersion(h.String())
 	return compareVersionsWithSuffix(v.Number, v.PatchLevel, v.Suffix, other)
 }
 
-// Implements compare.Eqer
-func (h HugoVersionString) Eq(other interface{}) bool {
+// Eq implements the compare.Eqer interface.
+func (h VersionString) Eq(other interface{}) bool {
 	s, err := cast.ToStringE(other)
 	if err != nil {
 		return false
@@ -70,9 +75,9 @@ func (h HugoVersionString) Eq(other interface{}) bool {
 
 var versionSuffixes = []string{"-test", "-DEV"}
 
-// ParseHugoVersion parses a version string.
-func ParseHugoVersion(s string) (HugoVersion, error) {
-	var vv HugoVersion
+// ParseVersion parses a version string.
+func ParseVersion(s string) (Version, error) {
+	var vv Version
 	for _, suffix := range versionSuffixes {
 		if strings.HasSuffix(s, suffix) {
 			vv.Suffix = suffix
@@ -88,10 +93,10 @@ func ParseHugoVersion(s string) (HugoVersion, error) {
 	return vv, nil
 }
 
-// MustParseHugoVersion parses a version string
+// MustParseVersion parses a version string
 // and panics if any error occurs.
-func MustParseHugoVersion(s string) HugoVersion {
-	vv, err := ParseHugoVersion(s)
+func MustParseVersion(s string) Version {
+	vv, err := ParseVersion(s)
 	if err != nil {
 		panic(err)
 	}
@@ -99,37 +104,53 @@ func MustParseHugoVersion(s string) HugoVersion {
 }
 
 // ReleaseVersion represents the release version.
-func (v HugoVersion) ReleaseVersion() HugoVersion {
+func (v Version) ReleaseVersion() Version {
 	v.Suffix = ""
 	return v
 }
 
 // Next returns the next Hugo release version.
-func (v HugoVersion) Next() HugoVersion {
-	return HugoVersion{Number: v.Number + 0.01}
+func (v Version) Next() Version {
+	return Version{Number: v.Number + 0.01}
 }
 
 // Prev returns the previous Hugo release version.
-func (v HugoVersion) Prev() HugoVersion {
-	return HugoVersion{Number: v.Number - 0.01}
+func (v Version) Prev() Version {
+	return Version{Number: v.Number - 0.01}
 }
 
 // NextPatchLevel returns the next patch/bugfix Hugo version.
 // This will be a patch increment on the previous Hugo version.
-func (v HugoVersion) NextPatchLevel(level int) HugoVersion {
-	return HugoVersion{Number: v.Number - 0.01, PatchLevel: level}
+func (v Version) NextPatchLevel(level int) Version {
+	return Version{Number: v.Number - 0.01, PatchLevel: level}
 }
 
-// CurrentHugoVersion represents the current build version.
-// This should be the only one.
-var CurrentHugoVersion = HugoVersion{
-	Number:     0.39,
-	PatchLevel: 0,
-	Suffix:     "",
+// BuildVersionString creates a version string. This is what you see when
+// running "hugo version".
+func BuildVersionString() string {
+	program := "Hugo Static Site Generator"
+
+	version := "v" + CurrentVersion.String()
+	if commitHash != "" {
+		version += "-" + strings.ToUpper(commitHash)
+	}
+	if IsExtended {
+		version += "/extended"
+	}
+
+	osArch := runtime.GOOS + "/" + runtime.GOARCH
+
+	date := buildDate
+	if date == "" {
+		date = "unknown"
+	}
+
+	return fmt.Sprintf("%s %s %s BuildDate: %s", program, version, osArch, date)
+
 }
 
-func hugoVersion(version float32, patchVersion int, suffix string) string {
-	if patchVersion > 0 {
+func version(version float32, patchVersion int, suffix string) string {
+	if patchVersion > 0 || version > 0.53 {
 		return fmt.Sprintf("%.2f.%d%s", version, patchVersion, suffix)
 	}
 	return fmt.Sprintf("%.2f%s", version, suffix)
@@ -140,7 +161,7 @@ func hugoVersion(version float32, patchVersion int, suffix string) string {
 // It returns -1 if the given version is less than, 0 if equal and 1 if greater than
 // the running version.
 func CompareVersion(version interface{}) int {
-	return compareVersionsWithSuffix(CurrentHugoVersion.Number, CurrentHugoVersion.PatchLevel, CurrentHugoVersion.Suffix, version)
+	return compareVersionsWithSuffix(CurrentVersion.Number, CurrentVersion.PatchLevel, CurrentVersion.Suffix, version)
 }
 
 func compareVersions(inVersion float32, inPatchVersion int, in interface{}) int {
@@ -166,7 +187,7 @@ func compareVersionsWithSuffix(inVersion float32, inPatchVersion int, suffix str
 			return -1
 		}
 
-		v, err := ParseHugoVersion(s)
+		v, err := ParseVersion(s)
 		if err != nil {
 			return -1
 		}
@@ -215,3 +236,16 @@ func compareFloatVersions(version float32, v float32) int {
 	}
 	return 1
 }
+
+func GoMinorVersion() int {
+	return goMinorVersion(runtime.Version())
+}
+
+func goMinorVersion(version string) int {
+	if strings.HasPrefix(version, "devel") {
+		return 9999 // magic
+	}
+	i, _ := strconv.Atoi(strings.Split(version, ".")[1])
+	return i
+
+}

common/hugo/version_current.go

@@ -0,0 +1,22 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugo
+
+// CurrentVersion represents the current build version.
+// This should be the only one.
+var CurrentVersion = Version{
+	Number:     0.57,
+	PatchLevel: 1,
+	Suffix:     "",
+}

common/hugo/version_test.go

@@ -0,0 +1,88 @@
+// Copyright 2015 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugo
+
+import (
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestHugoVersion(t *testing.T) {
+	c := qt.New(t)
+
+	c.Assert(version(0.15, 0, "-DEV"), qt.Equals, "0.15-DEV")
+	c.Assert(version(0.15, 2, "-DEV"), qt.Equals, "0.15.2-DEV")
+
+	v := Version{Number: 0.21, PatchLevel: 0, Suffix: "-DEV"}
+
+	c.Assert(v.ReleaseVersion().String(), qt.Equals, "0.21")
+	c.Assert(v.String(), qt.Equals, "0.21-DEV")
+	c.Assert(v.Next().String(), qt.Equals, "0.22")
+	nextVersionString := v.Next().Version()
+	c.Assert(nextVersionString.String(), qt.Equals, "0.22")
+	c.Assert(nextVersionString.Eq("0.22"), qt.Equals, true)
+	c.Assert(nextVersionString.Eq("0.21"), qt.Equals, false)
+	c.Assert(nextVersionString.Eq(nextVersionString), qt.Equals, true)
+	c.Assert(v.NextPatchLevel(3).String(), qt.Equals, "0.20.3")
+
+	// We started to use full semver versions even for main
+	// releases in v0.54.0
+	v = Version{Number: 0.53, PatchLevel: 0}
+	c.Assert(v.String(), qt.Equals, "0.53")
+	c.Assert(v.Next().String(), qt.Equals, "0.54.0")
+	c.Assert(v.Next().Next().String(), qt.Equals, "0.55.0")
+	v = Version{Number: 0.54, PatchLevel: 0, Suffix: "-DEV"}
+	c.Assert(v.String(), qt.Equals, "0.54.0-DEV")
+}
+
+func TestCompareVersions(t *testing.T) {
+	c := qt.New(t)
+
+	c.Assert(compareVersions(0.20, 0, 0.20), qt.Equals, 0)
+	c.Assert(compareVersions(0.20, 0, float32(0.20)), qt.Equals, 0)
+	c.Assert(compareVersions(0.20, 0, float64(0.20)), qt.Equals, 0)
+	c.Assert(compareVersions(0.19, 1, 0.20), qt.Equals, 1)
+	c.Assert(compareVersions(0.19, 3, "0.20.2"), qt.Equals, 1)
+	c.Assert(compareVersions(0.19, 1, 0.01), qt.Equals, -1)
+	c.Assert(compareVersions(0, 1, 3), qt.Equals, 1)
+	c.Assert(compareVersions(0, 1, int32(3)), qt.Equals, 1)
+	c.Assert(compareVersions(0, 1, int64(3)), qt.Equals, 1)
+	c.Assert(compareVersions(0.20, 0, "0.20"), qt.Equals, 0)
+	c.Assert(compareVersions(0.20, 1, "0.20.1"), qt.Equals, 0)
+	c.Assert(compareVersions(0.20, 1, "0.20"), qt.Equals, -1)
+	c.Assert(compareVersions(0.20, 0, "0.20.1"), qt.Equals, 1)
+	c.Assert(compareVersions(0.20, 1, "0.20.2"), qt.Equals, 1)
+	c.Assert(compareVersions(0.21, 1, "0.22.1"), qt.Equals, 1)
+	c.Assert(compareVersions(0.22, 0, "0.22-DEV"), qt.Equals, -1)
+	c.Assert(compareVersions(0.22, 0, "0.22.1-DEV"), qt.Equals, 1)
+	c.Assert(compareVersionsWithSuffix(0.22, 0, "-DEV", "0.22"), qt.Equals, 1)
+	c.Assert(compareVersionsWithSuffix(0.22, 1, "-DEV", "0.22"), qt.Equals, -1)
+	c.Assert(compareVersionsWithSuffix(0.22, 1, "-DEV", "0.22.1-DEV"), qt.Equals, 0)
+}
+
+func TestParseHugoVersion(t *testing.T) {
+	c := qt.New(t)
+
+	c.Assert(MustParseVersion("0.25").String(), qt.Equals, "0.25")
+	c.Assert(MustParseVersion("0.25.2").String(), qt.Equals, "0.25.2")
+	c.Assert(MustParseVersion("0.25-test").String(), qt.Equals, "0.25-test")
+	c.Assert(MustParseVersion("0.25-DEV").String(), qt.Equals, "0.25-DEV")
+}
+
+func TestGoMinorVersion(t *testing.T) {
+	c := qt.New(t)
+	c.Assert(goMinorVersion("go1.12.5"), qt.Equals, 12)
+	c.Assert(GoMinorVersion() >= 11, qt.Equals, true)
+}

common/loggers/loggers.go

@@ -0,0 +1,176 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package loggers
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"regexp"
+
+	"github.com/gohugoio/hugo/common/terminal"
+
+	jww "github.com/spf13/jwalterweatherman"
+)
+
+var (
+	// Counts ERROR logs to the global jww logger.
+	GlobalErrorCounter *jww.Counter
+)
+
+func init() {
+	GlobalErrorCounter = &jww.Counter{}
+	jww.SetLogListeners(jww.LogCounter(GlobalErrorCounter, jww.LevelError))
+}
+
+// Logger wraps a *loggers.Logger and some other related logging state.
+type Logger struct {
+	*jww.Notepad
+	ErrorCounter *jww.Counter
+	WarnCounter  *jww.Counter
+
+	// This is only set in server mode.
+	errors *bytes.Buffer
+}
+
+func (l *Logger) Errors() string {
+	if l.errors == nil {
+		return ""
+	}
+	return ansiColorRe.ReplaceAllString(l.errors.String(), "")
+}
+
+// Reset resets the logger's internal state.
+func (l *Logger) Reset() {
+	l.ErrorCounter.Reset()
+	if l.errors != nil {
+		l.errors.Reset()
+	}
+}
+
+//  NewLogger creates a new Logger for the given thresholds
+func NewLogger(stdoutThreshold, logThreshold jww.Threshold, outHandle, logHandle io.Writer, saveErrors bool) *Logger {
+	return newLogger(stdoutThreshold, logThreshold, outHandle, logHandle, saveErrors)
+}
+
+// NewDebugLogger is a convenience function to create a debug logger.
+func NewDebugLogger() *Logger {
+	return newBasicLogger(jww.LevelDebug)
+}
+
+// NewWarningLogger is a convenience function to create a warning logger.
+func NewWarningLogger() *Logger {
+	return newBasicLogger(jww.LevelWarn)
+}
+
+// NewErrorLogger is a convenience function to create an error logger.
+func NewErrorLogger() *Logger {
+	return newBasicLogger(jww.LevelError)
+}
+
+var (
+	ansiColorRe = regexp.MustCompile("(?s)\\033\\[\\d*(;\\d*)*m")
+	errorRe     = regexp.MustCompile("^(ERROR|FATAL|WARN)")
+)
+
+type ansiCleaner struct {
+	w io.Writer
+}
+
+func (a ansiCleaner) Write(p []byte) (n int, err error) {
+	return a.w.Write(ansiColorRe.ReplaceAll(p, []byte("")))
+}
+
+type labelColorizer struct {
+	w io.Writer
+}
+
+func (a labelColorizer) Write(p []byte) (n int, err error) {
+	replaced := errorRe.ReplaceAllStringFunc(string(p), func(m string) string {
+		switch m {
+		case "ERROR", "FATAL":
+			return terminal.Error(m)
+		case "WARN":
+			return terminal.Warning(m)
+		default:
+			return m
+		}
+	})
+	// io.MultiWriter will abort if we return a bigger write count than input
+	// bytes, so we lie a little.
+	_, err = a.w.Write([]byte(replaced))
+	return len(p), err
+
+}
+
+// InitGlobalLogger initializes the global logger, used in some rare cases.
+func InitGlobalLogger(stdoutThreshold, logThreshold jww.Threshold, outHandle, logHandle io.Writer) {
+	outHandle, logHandle = getLogWriters(outHandle, logHandle)
+
+	jww.SetStdoutOutput(outHandle)
+	jww.SetLogOutput(logHandle)
+	jww.SetLogThreshold(logThreshold)
+	jww.SetStdoutThreshold(stdoutThreshold)
+
+}
+
+func getLogWriters(outHandle, logHandle io.Writer) (io.Writer, io.Writer) {
+	isTerm := terminal.IsTerminal(os.Stdout)
+	if logHandle != ioutil.Discard && isTerm {
+		// Remove any Ansi coloring from log output
+		logHandle = ansiCleaner{w: logHandle}
+	}
+
+	if isTerm {
+		outHandle = labelColorizer{w: outHandle}
+	}
+
+	return outHandle, logHandle
+
+}
+
+func newLogger(stdoutThreshold, logThreshold jww.Threshold, outHandle, logHandle io.Writer, saveErrors bool) *Logger {
+	errorCounter := &jww.Counter{}
+	warnCounter := &jww.Counter{}
+	outHandle, logHandle = getLogWriters(outHandle, logHandle)
+
+	listeners := []jww.LogListener{jww.LogCounter(errorCounter, jww.LevelError), jww.LogCounter(warnCounter, jww.LevelWarn)}
+	var errorBuff *bytes.Buffer
+	if saveErrors {
+		errorBuff = new(bytes.Buffer)
+		errorCapture := func(t jww.Threshold) io.Writer {
+			if t != jww.LevelError {
+				// Only interested in ERROR
+				return nil
+			}
+
+			return errorBuff
+		}
+
+		listeners = append(listeners, errorCapture)
+	}
+
+	return &Logger{
+		Notepad:      jww.NewNotepad(stdoutThreshold, logThreshold, outHandle, logHandle, "", log.Ldate|log.Ltime, listeners...),
+		ErrorCounter: errorCounter,
+		WarnCounter:  warnCounter,
+		errors:       errorBuff,
+	}
+}
+
+func newBasicLogger(t jww.Threshold) *Logger {
+	return newLogger(t, jww.LevelError, os.Stdout, ioutil.Discard, false)
+}

common/loggers/loggers_test.go

@@ -11,23 +11,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package hugolib
+package loggers
 
 import (
-	"fmt"
 	"testing"
 
-	"github.com/gohugoio/hugo/helpers"
-	"github.com/stretchr/testify/require"
+	qt "github.com/frankban/quicktest"
 )
 
-func TestHugoInfo(t *testing.T) {
-	assert := require.New(t)
+func TestLogger(t *testing.T) {
+	c := qt.New(t)
+	l := NewWarningLogger()
 
-	assert.Equal(helpers.CurrentHugoVersion.Version(), hugoInfo.Version)
-	assert.IsType(helpers.HugoVersionString(""), hugoInfo.Version)
-	assert.Equal(CommitHash, hugoInfo.CommitHash)
-	assert.Equal(BuildDate, hugoInfo.BuildDate)
-	assert.Contains(hugoInfo.Generator, fmt.Sprintf("Hugo %s", hugoInfo.Version))
+	l.ERROR.Println("One error")
+	l.ERROR.Println("Two error")
+	l.WARN.Println("A warning")
+
+	c.Assert(l.ErrorCounter.Count(), qt.Equals, uint64(2))
 
 }

common/maps/maps.go

@@ -0,0 +1,116 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package maps
+
+import (
+	"strings"
+
+	"github.com/gobwas/glob"
+
+	"github.com/spf13/cast"
+)
+
+// ToLower makes all the keys in the given map lower cased and will do so
+// recursively.
+// Notes:
+// * This will modify the map given.
+// * Any nested map[interface{}]interface{} will be converted to map[string]interface{}.
+func ToLower(m map[string]interface{}) {
+	for k, v := range m {
+		switch v.(type) {
+		case map[interface{}]interface{}:
+			v = cast.ToStringMap(v)
+			ToLower(v.(map[string]interface{}))
+		case map[string]interface{}:
+			ToLower(v.(map[string]interface{}))
+		}
+
+		lKey := strings.ToLower(k)
+		if k != lKey {
+			delete(m, k)
+			m[lKey] = v
+		}
+
+	}
+}
+
+type keyRename struct {
+	pattern glob.Glob
+	newKey  string
+}
+
+// KeyRenamer supports renaming of keys in a map.
+type KeyRenamer struct {
+	renames []keyRename
+}
+
+// NewKeyRenamer creates a new KeyRenamer given a list of pattern and new key
+// value pairs.
+func NewKeyRenamer(patternKeys ...string) (KeyRenamer, error) {
+	var renames []keyRename
+	for i := 0; i < len(patternKeys); i += 2 {
+		g, err := glob.Compile(strings.ToLower(patternKeys[i]), '/')
+		if err != nil {
+			return KeyRenamer{}, err
+		}
+		renames = append(renames, keyRename{pattern: g, newKey: patternKeys[i+1]})
+	}
+
+	return KeyRenamer{renames: renames}, nil
+}
+
+func (r KeyRenamer) getNewKey(keyPath string) string {
+	for _, matcher := range r.renames {
+		if matcher.pattern.Match(keyPath) {
+			return matcher.newKey
+		}
+	}
+
+	return ""
+}
+
+// Rename renames the keys in the given map according
+// to the patterns in the current KeyRenamer.
+func (r KeyRenamer) Rename(m map[string]interface{}) {
+	r.renamePath("", m)
+}
+
+func (KeyRenamer) keyPath(k1, k2 string) string {
+	k1, k2 = strings.ToLower(k1), strings.ToLower(k2)
+	if k1 == "" {
+		return k2
+	} else {
+		return k1 + "/" + k2
+	}
+}
+
+func (r KeyRenamer) renamePath(parentKeyPath string, m map[string]interface{}) {
+	for key, val := range m {
+		keyPath := r.keyPath(parentKeyPath, key)
+		switch val.(type) {
+		case map[interface{}]interface{}:
+			val = cast.ToStringMap(val)
+			r.renamePath(keyPath, val.(map[string]interface{}))
+		case map[string]interface{}:
+			r.renamePath(keyPath, val.(map[string]interface{}))
+		}
+
+		newKey := r.getNewKey(keyPath)
+
+		if newKey != "" {
+			delete(m, key)
+			m[newKey] = val
+		}
+	}
+}

common/maps/maps_get.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,18 +11,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build linux darwin !windows
+package maps
 
-package source
+import (
+	"github.com/spf13/cast"
+)
 
-//
-// NOTE, any changes here need to be reflected in filesystem_windows_test.go
-//
-var platformBase = "/base/"
-var platformPaths = []TestPath{
-	{"foobar", "foobar", "aaa", "", ""},
-	{"b/1file", "1file", "aaa", "b", "b/"},
-	{"c/d/2file", "2file", "aaa", "c", "c/d/"},
-	{"/base/e/f/3file", "3file", "aaa", "e", "e/f/"},
-	{"section/foo.rss", "foo.rss", "aaa", "section", "section/"},
+// GetString tries to get a value with key from map m and convert it to a string.
+// It will return an empty string if not found or if it cannot be convertd to a string.
+func GetString(m map[string]interface{}, key string) string {
+	if m == nil {
+		return ""
+	}
+	v, found := m[key]
+	if !found {
+		return ""
+	}
+	return cast.ToString(v)
 }

common/maps/maps_test.go

@@ -0,0 +1,123 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package maps
+
+import (
+	"reflect"
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestToLower(t *testing.T) {
+
+	tests := []struct {
+		input    map[string]interface{}
+		expected map[string]interface{}
+	}{
+		{
+			map[string]interface{}{
+				"abC": 32,
+			},
+			map[string]interface{}{
+				"abc": 32,
+			},
+		},
+		{
+			map[string]interface{}{
+				"abC": 32,
+				"deF": map[interface{}]interface{}{
+					23: "A value",
+					24: map[string]interface{}{
+						"AbCDe": "A value",
+						"eFgHi": "Another value",
+					},
+				},
+				"gHi": map[string]interface{}{
+					"J": 25,
+				},
+			},
+			map[string]interface{}{
+				"abc": 32,
+				"def": map[string]interface{}{
+					"23": "A value",
+					"24": map[string]interface{}{
+						"abcde": "A value",
+						"efghi": "Another value",
+					},
+				},
+				"ghi": map[string]interface{}{
+					"j": 25,
+				},
+			},
+		},
+	}
+
+	for i, test := range tests {
+		// ToLower modifies input.
+		ToLower(test.input)
+		if !reflect.DeepEqual(test.expected, test.input) {
+			t.Errorf("[%d] Expected\n%#v, got\n%#v\n", i, test.expected, test.input)
+		}
+	}
+}
+
+func TestRenameKeys(t *testing.T) {
+	c := qt.New(t)
+
+	m := map[string]interface{}{
+		"a":    32,
+		"ren1": "m1",
+		"ren2": "m1_2",
+		"sub": map[string]interface{}{
+			"subsub": map[string]interface{}{
+				"REN1": "m2",
+				"ren2": "m2_2",
+			},
+		},
+		"no": map[string]interface{}{
+			"ren1": "m2",
+			"ren2": "m2_2",
+		},
+	}
+
+	expected := map[string]interface{}{
+		"a":    32,
+		"new1": "m1",
+		"new2": "m1_2",
+		"sub": map[string]interface{}{
+			"subsub": map[string]interface{}{
+				"new1": "m2",
+				"ren2": "m2_2",
+			},
+		},
+		"no": map[string]interface{}{
+			"ren1": "m2",
+			"ren2": "m2_2",
+		},
+	}
+
+	renamer, err := NewKeyRenamer(
+		"{ren1,sub/*/ren1}", "new1",
+		"{Ren2,sub/ren2}", "new2",
+	)
+	c.Assert(err, qt.IsNil)
+
+	renamer.Rename(m)
+
+	if !reflect.DeepEqual(expected, m) {
+		t.Errorf("Expected\n%#v, got\n%#v\n", expected, m)
+	}
+
+}

common/maps/params.go

@@ -0,0 +1,94 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package maps
+
+import (
+	"strings"
+
+	"github.com/spf13/cast"
+)
+
+// GetNestedParam gets the first match of the keyStr in the candidates given.
+// It will first try the exact match and then try to find it as a nested map value,
+// using the given separator, e.g. "mymap.name".
+// It assumes that all the maps given have lower cased keys.
+func GetNestedParam(keyStr, separator string, candidates ...map[string]interface{}) (interface{}, error) {
+	keyStr = strings.ToLower(keyStr)
+
+	lookupFn := func(key string) interface{} {
+		for _, m := range candidates {
+			if v, ok := m[key]; ok {
+				return v
+			}
+		}
+
+		return nil
+	}
+
+	v, _, _, err := GetNestedParamFn(keyStr, separator, lookupFn)
+	return v, err
+}
+
+func GetNestedParamFn(keyStr, separator string, lookupFn func(key string) interface{}) (interface{}, string, map[string]interface{}, error) {
+	result, _ := traverseDirectParams(keyStr, lookupFn)
+	if result != nil {
+		return result, keyStr, nil, nil
+	}
+
+	keySegments := strings.Split(keyStr, separator)
+	if len(keySegments) == 1 {
+		return nil, keyStr, nil, nil
+	}
+
+	return traverseNestedParams(keySegments, lookupFn)
+}
+
+func traverseDirectParams(keyStr string, lookupFn func(key string) interface{}) (interface{}, error) {
+	return lookupFn(keyStr), nil
+}
+
+func traverseNestedParams(keySegments []string, lookupFn func(key string) interface{}) (interface{}, string, map[string]interface{}, error) {
+	firstKey, rest := keySegments[0], keySegments[1:]
+	result := lookupFn(firstKey)
+	if result == nil || len(rest) == 0 {
+		return result, firstKey, nil, nil
+	}
+
+	switch m := result.(type) {
+	case map[string]interface{}:
+		v, key, owner := traverseParams(rest, m)
+		return v, key, owner, nil
+	default:
+		return nil, "", nil, nil
+	}
+}
+
+func traverseParams(keys []string, m map[string]interface{}) (interface{}, string, map[string]interface{}) {
+	// Shift first element off.
+	firstKey, rest := keys[0], keys[1:]
+	result := m[firstKey]
+
+	// No point in continuing here.
+	if result == nil {
+		return result, "", nil
+	}
+
+	if len(rest) == 0 {
+		// That was the last key.
+		return result, firstKey, m
+	}
+
+	// That was not the last key.
+	return traverseParams(rest, cast.ToStringMap(result))
+}

common/maps/params_test.go

@@ -0,0 +1,51 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package maps
+
+import (
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestGetNestedParam(t *testing.T) {
+
+	m := map[string]interface{}{
+		"string":          "value",
+		"first":           1,
+		"with_underscore": 2,
+		"nested": map[string]interface{}{
+			"color": "blue",
+			"nestednested": map[string]interface{}{
+				"color": "green",
+			},
+		},
+	}
+
+	c := qt.New(t)
+
+	must := func(keyStr, separator string, candidates ...map[string]interface{}) interface{} {
+		v, err := GetNestedParam(keyStr, separator, candidates...)
+		c.Assert(err, qt.IsNil)
+		return v
+	}
+
+	c.Assert(must("first", "_", m), qt.Equals, 1)
+	c.Assert(must("First", "_", m), qt.Equals, 1)
+	c.Assert(must("with_underscore", "_", m), qt.Equals, 2)
+	c.Assert(must("nested_color", "_", m), qt.Equals, "blue")
+	c.Assert(must("nested.nestednested.color", ".", m), qt.Equals, "green")
+	c.Assert(must("string.name", ".", m), qt.IsNil)
+
+}

common/maps/scratch.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,14 +11,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package hugolib
+package maps
 
 import (
 	"reflect"
 	"sort"
 	"sync"
 
-	"github.com/gohugoio/hugo/tpl/math"
+	"github.com/gohugoio/hugo/common/collections"
+	"github.com/gohugoio/hugo/common/math"
 )
 
 // Scratch is a writable context used for stateful operations in Page/Node rendering.
@@ -27,6 +28,24 @@ type Scratch struct {
 	mu     sync.RWMutex
 }
 
+// Scratcher provides a scratching service.
+type Scratcher interface {
+	Scratch() *Scratch
+}
+
+type scratcher struct {
+	s *Scratch
+}
+
+func (s scratcher) Scratch() *Scratch {
+	return s.s
+}
+
+// NewScratcher creates a new Scratcher.
+func NewScratcher() Scratcher {
+	return scratcher{s: NewScratch()}
+}
+
 // Add will, for single values, add (using the + operator) the addend to the existing addend (if found).
 // Supports numeric values and strings.
 //
@@ -40,14 +59,12 @@ func (c *Scratch) Add(key string, newAddend interface{}) (string, error) {
 	if found {
 		var err error
 
-		addendV := reflect.ValueOf(existingAddend)
+		addendV := reflect.TypeOf(existingAddend)
 
 		if addendV.Kind() == reflect.Slice || addendV.Kind() == reflect.Array {
-			nav := reflect.ValueOf(newAddend)
-			if nav.Kind() == reflect.Slice || nav.Kind() == reflect.Array {
-				newVal = reflect.AppendSlice(addendV, nav).Interface()
-			} else {
-				newVal = reflect.Append(addendV, nav).Interface()
+			newVal, err = collections.Append(existingAddend, newAddend)
+			if err != nil {
+				return "", err
 			}
 		} else {
 			newVal, err = math.DoArithmetic(existingAddend, newAddend, '+')
@@ -73,7 +90,7 @@ func (c *Scratch) Set(key string, value interface{}) string {
 	return ""
 }
 
-// Reset deletes the given key
+// Delete deletes the given key.
 func (c *Scratch) Delete(key string) string {
 	c.mu.Lock()
 	delete(c.values, key)
@@ -81,7 +98,7 @@ func (c *Scratch) Delete(key string) string {
 	return ""
 }
 
-// Get returns a value previously set by Add or Set
+// Get returns a value previously set by Add or Set.
 func (c *Scratch) Get(key string) interface{} {
 	c.mu.RLock()
 	val := c.values[key]
@@ -104,7 +121,7 @@ func (c *Scratch) SetInMap(key string, mapKey string, value interface{}) string
 	return ""
 }
 
-// GetSortedMapValues returns a sorted map previously filled with SetInMap
+// GetSortedMapValues returns a sorted map previously filled with SetInMap.
 func (c *Scratch) GetSortedMapValues(key string) interface{} {
 	c.mu.RLock()
 
@@ -130,6 +147,7 @@ func (c *Scratch) GetSortedMapValues(key string) interface{} {
 	return sortedArray
 }
 
-func newScratch() *Scratch {
+// NewScratch returns a new instance Scratch.
+func NewScratch() *Scratch {
 	return &Scratch{values: make(map[string]interface{})}
 }

common/maps/scratch_test.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,36 +11,38 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package hugolib
+package maps
 
 import (
 	"reflect"
 	"sync"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
+	qt "github.com/frankban/quicktest"
 )
 
 func TestScratchAdd(t *testing.T) {
 	t.Parallel()
-	scratch := newScratch()
+	c := qt.New(t)
+
+	scratch := NewScratch()
 	scratch.Add("int1", 10)
 	scratch.Add("int1", 20)
 	scratch.Add("int2", 20)
 
-	assert.Equal(t, int64(30), scratch.Get("int1"))
-	assert.Equal(t, 20, scratch.Get("int2"))
+	c.Assert(scratch.Get("int1"), qt.Equals, int64(30))
+	c.Assert(scratch.Get("int2"), qt.Equals, 20)
 
 	scratch.Add("float1", float64(10.5))
 	scratch.Add("float1", float64(20.1))
 
-	assert.Equal(t, float64(30.6), scratch.Get("float1"))
+	c.Assert(scratch.Get("float1"), qt.Equals, float64(30.6))
 
 	scratch.Add("string1", "Hello ")
 	scratch.Add("string1", "big ")
 	scratch.Add("string1", "World!")
 
-	assert.Equal(t, "Hello big World!", scratch.Get("string1"))
+	c.Assert(scratch.Get("string1"), qt.Equals, "Hello big World!")
 
 	scratch.Add("scratch", scratch)
 	_, err := scratch.Add("scratch", scratch)
@@ -53,12 +55,14 @@ func TestScratchAdd(t *testing.T) {
 
 func TestScratchAddSlice(t *testing.T) {
 	t.Parallel()
-	scratch := newScratch()
+	c := qt.New(t)
+
+	scratch := NewScratch()
 
 	_, err := scratch.Add("intSlice", []int{1, 2})
-	assert.Nil(t, err)
+	c.Assert(err, qt.IsNil)
 	_, err = scratch.Add("intSlice", 3)
-	assert.Nil(t, err)
+	c.Assert(err, qt.IsNil)
 
 	sl := scratch.Get("intSlice")
 	expected := []int{1, 2, 3}
@@ -66,10 +70,9 @@ func TestScratchAddSlice(t *testing.T) {
 	if !reflect.DeepEqual(expected, sl) {
 		t.Errorf("Slice difference, go %q expected %q", sl, expected)
 	}
-
 	_, err = scratch.Add("intSlice", []int{4, 5})
 
-	assert.Nil(t, err)
+	c.Assert(err, qt.IsNil)
 
 	sl = scratch.Get("intSlice")
 	expected = []int{1, 2, 3, 4, 5}
@@ -77,29 +80,61 @@ func TestScratchAddSlice(t *testing.T) {
 	if !reflect.DeepEqual(expected, sl) {
 		t.Errorf("Slice difference, go %q expected %q", sl, expected)
 	}
+}
+
+// https://github.com/gohugoio/hugo/issues/5275
+func TestScratchAddTypedSliceToInterfaceSlice(t *testing.T) {
+	t.Parallel()
+	c := qt.New(t)
+
+	scratch := NewScratch()
+	scratch.Set("slice", []interface{}{})
+
+	_, err := scratch.Add("slice", []int{1, 2})
+	c.Assert(err, qt.IsNil)
+	c.Assert(scratch.Get("slice"), qt.DeepEquals, []int{1, 2})
+
+}
+
+// https://github.com/gohugoio/hugo/issues/5361
+func TestScratchAddDifferentTypedSliceToInterfaceSlice(t *testing.T) {
+	t.Parallel()
+	c := qt.New(t)
+
+	scratch := NewScratch()
+	scratch.Set("slice", []string{"foo"})
+
+	_, err := scratch.Add("slice", []int{1, 2})
+	c.Assert(err, qt.IsNil)
+	c.Assert(scratch.Get("slice"), qt.DeepEquals, []interface{}{"foo", 1, 2})
 
 }
 
 func TestScratchSet(t *testing.T) {
 	t.Parallel()
-	scratch := newScratch()
+	c := qt.New(t)
+
+	scratch := NewScratch()
 	scratch.Set("key", "val")
-	assert.Equal(t, "val", scratch.Get("key"))
+	c.Assert(scratch.Get("key"), qt.Equals, "val")
 }
 
 func TestScratchDelete(t *testing.T) {
 	t.Parallel()
-	scratch := newScratch()
+	c := qt.New(t)
+
+	scratch := NewScratch()
 	scratch.Set("key", "val")
 	scratch.Delete("key")
 	scratch.Add("key", "Lucy Parsons")
-	assert.Equal(t, "Lucy Parsons", scratch.Get("key"))
+	c.Assert(scratch.Get("key"), qt.Equals, "Lucy Parsons")
 }
 
 // Issue #2005
 func TestScratchInParallel(t *testing.T) {
 	var wg sync.WaitGroup
-	scratch := newScratch()
+	scratch := NewScratch()
+
 	key := "counter"
 	scratch.Set(key, int64(1))
 	for i := 1; i <= 10; i++ {
@@ -133,7 +168,7 @@ func TestScratchInParallel(t *testing.T) {
 
 func TestScratchGet(t *testing.T) {
 	t.Parallel()
-	scratch := newScratch()
+	scratch := NewScratch()
 	nothing := scratch.Get("nothing")
 	if nothing != nil {
 		t.Errorf("Should not return anything, but got %v", nothing)
@@ -142,18 +177,20 @@ func TestScratchGet(t *testing.T) {
 
 func TestScratchSetInMap(t *testing.T) {
 	t.Parallel()
-	scratch := newScratch()
+	c := qt.New(t)
+
+	scratch := NewScratch()
 	scratch.SetInMap("key", "lux", "Lux")
 	scratch.SetInMap("key", "abc", "Abc")
 	scratch.SetInMap("key", "zyx", "Zyx")
 	scratch.SetInMap("key", "abc", "Abc (updated)")
 	scratch.SetInMap("key", "def", "Def")
-	assert.Equal(t, []interface{}{0: "Abc (updated)", 1: "Def", 2: "Lux", 3: "Zyx"}, scratch.GetSortedMapValues("key"))
+	c.Assert(scratch.GetSortedMapValues("key"), qt.DeepEquals, []interface{}{0: "Abc (updated)", 1: "Def", 2: "Lux", 3: "Zyx"})
 }
 
 func TestScratchGetSortedMapValues(t *testing.T) {
 	t.Parallel()
-	scratch := newScratch()
+	scratch := NewScratch()
 	nothing := scratch.GetSortedMapValues("nothing")
 	if nothing != nil {
 		t.Errorf("Should not return anything, but got %v", nothing)
@@ -161,7 +198,7 @@ func TestScratchGetSortedMapValues(t *testing.T) {
 }
 
 func BenchmarkScratchGet(b *testing.B) {
-	scratch := newScratch()
+	scratch := NewScratch()
 	scratch.Add("A", 1)
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {

common/math/math.go

@@ -0,0 +1,135 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package math
+
+import (
+	"errors"
+	"reflect"
+)
+
+// DoArithmetic performs arithmetic operations (+,-,*,/) using reflection to
+// determine the type of the two terms.
+func DoArithmetic(a, b interface{}, op rune) (interface{}, error) {
+	av := reflect.ValueOf(a)
+	bv := reflect.ValueOf(b)
+	var ai, bi int64
+	var af, bf float64
+	var au, bu uint64
+	switch av.Kind() {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		ai = av.Int()
+		switch bv.Kind() {
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			bi = bv.Int()
+		case reflect.Float32, reflect.Float64:
+			af = float64(ai) // may overflow
+			ai = 0
+			bf = bv.Float()
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			bu = bv.Uint()
+			if ai >= 0 {
+				au = uint64(ai)
+				ai = 0
+			} else {
+				bi = int64(bu) // may overflow
+				bu = 0
+			}
+		default:
+			return nil, errors.New("can't apply the operator to the values")
+		}
+	case reflect.Float32, reflect.Float64:
+		af = av.Float()
+		switch bv.Kind() {
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			bf = float64(bv.Int()) // may overflow
+		case reflect.Float32, reflect.Float64:
+			bf = bv.Float()
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			bf = float64(bv.Uint()) // may overflow
+		default:
+			return nil, errors.New("can't apply the operator to the values")
+		}
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		au = av.Uint()
+		switch bv.Kind() {
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			bi = bv.Int()
+			if bi >= 0 {
+				bu = uint64(bi)
+				bi = 0
+			} else {
+				ai = int64(au) // may overflow
+				au = 0
+			}
+		case reflect.Float32, reflect.Float64:
+			af = float64(au) // may overflow
+			au = 0
+			bf = bv.Float()
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			bu = bv.Uint()
+		default:
+			return nil, errors.New("can't apply the operator to the values")
+		}
+	case reflect.String:
+		as := av.String()
+		if bv.Kind() == reflect.String && op == '+' {
+			bs := bv.String()
+			return as + bs, nil
+		}
+		return nil, errors.New("can't apply the operator to the values")
+	default:
+		return nil, errors.New("can't apply the operator to the values")
+	}
+
+	switch op {
+	case '+':
+		if ai != 0 || bi != 0 {
+			return ai + bi, nil
+		} else if af != 0 || bf != 0 {
+			return af + bf, nil
+		} else if au != 0 || bu != 0 {
+			return au + bu, nil
+		}
+		return 0, nil
+	case '-':
+		if ai != 0 || bi != 0 {
+			return ai - bi, nil
+		} else if af != 0 || bf != 0 {
+			return af - bf, nil
+		} else if au != 0 || bu != 0 {
+			return au - bu, nil
+		}
+		return 0, nil
+	case '*':
+		if ai != 0 || bi != 0 {
+			return ai * bi, nil
+		} else if af != 0 || bf != 0 {
+			return af * bf, nil
+		} else if au != 0 || bu != 0 {
+			return au * bu, nil
+		}
+		return 0, nil
+	case '/':
+		if bi != 0 {
+			return ai / bi, nil
+		} else if bf != 0 {
+			return af / bf, nil
+		} else if bu != 0 {
+			return au / bu, nil
+		}
+		return nil, errors.New("can't divide the value by 0")
+	default:
+		return nil, errors.New("there is no such an operation")
+	}
+}

common/math/math_test.go

@@ -0,0 +1,106 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package math
+
+import (
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestDoArithmetic(t *testing.T) {
+	t.Parallel()
+	c := qt.New(t)
+
+	for _, test := range []struct {
+		a      interface{}
+		b      interface{}
+		op     rune
+		expect interface{}
+	}{
+		{3, 2, '+', int64(5)},
+		{3, 2, '-', int64(1)},
+		{3, 2, '*', int64(6)},
+		{3, 2, '/', int64(1)},
+		{3.0, 2, '+', float64(5)},
+		{3.0, 2, '-', float64(1)},
+		{3.0, 2, '*', float64(6)},
+		{3.0, 2, '/', float64(1.5)},
+		{3, 2.0, '+', float64(5)},
+		{3, 2.0, '-', float64(1)},
+		{3, 2.0, '*', float64(6)},
+		{3, 2.0, '/', float64(1.5)},
+		{3.0, 2.0, '+', float64(5)},
+		{3.0, 2.0, '-', float64(1)},
+		{3.0, 2.0, '*', float64(6)},
+		{3.0, 2.0, '/', float64(1.5)},
+		{uint(3), uint(2), '+', uint64(5)},
+		{uint(3), uint(2), '-', uint64(1)},
+		{uint(3), uint(2), '*', uint64(6)},
+		{uint(3), uint(2), '/', uint64(1)},
+		{uint(3), 2, '+', uint64(5)},
+		{uint(3), 2, '-', uint64(1)},
+		{uint(3), 2, '*', uint64(6)},
+		{uint(3), 2, '/', uint64(1)},
+		{3, uint(2), '+', uint64(5)},
+		{3, uint(2), '-', uint64(1)},
+		{3, uint(2), '*', uint64(6)},
+		{3, uint(2), '/', uint64(1)},
+		{uint(3), -2, '+', int64(1)},
+		{uint(3), -2, '-', int64(5)},
+		{uint(3), -2, '*', int64(-6)},
+		{uint(3), -2, '/', int64(-1)},
+		{-3, uint(2), '+', int64(-1)},
+		{-3, uint(2), '-', int64(-5)},
+		{-3, uint(2), '*', int64(-6)},
+		{-3, uint(2), '/', int64(-1)},
+		{uint(3), 2.0, '+', float64(5)},
+		{uint(3), 2.0, '-', float64(1)},
+		{uint(3), 2.0, '*', float64(6)},
+		{uint(3), 2.0, '/', float64(1.5)},
+		{3.0, uint(2), '+', float64(5)},
+		{3.0, uint(2), '-', float64(1)},
+		{3.0, uint(2), '*', float64(6)},
+		{3.0, uint(2), '/', float64(1.5)},
+		{0, 0, '+', 0},
+		{0, 0, '-', 0},
+		{0, 0, '*', 0},
+		{"foo", "bar", '+', "foobar"},
+		{3, 0, '/', false},
+		{3.0, 0, '/', false},
+		{3, 0.0, '/', false},
+		{uint(3), uint(0), '/', false},
+		{3, uint(0), '/', false},
+		{-3, uint(0), '/', false},
+		{uint(3), 0, '/', false},
+		{3.0, uint(0), '/', false},
+		{uint(3), 0.0, '/', false},
+		{3, "foo", '+', false},
+		{3.0, "foo", '+', false},
+		{uint(3), "foo", '+', false},
+		{"foo", 3, '+', false},
+		{"foo", "bar", '-', false},
+		{3, 2, '%', false},
+	} {
+		result, err := DoArithmetic(test.a, test.b, test.op)
+
+		if b, ok := test.expect.(bool); ok && !b {
+			c.Assert(err, qt.Not(qt.IsNil))
+			continue
+		}
+
+		c.Assert(err, qt.IsNil)
+		c.Assert(test.expect, qt.Equals, result)
+	}
+}

common/terminal/colors.go

@@ -0,0 +1,70 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package terminal contains helper for the terminal, such as coloring output.
+package terminal
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"strings"
+
+	isatty "github.com/mattn/go-isatty"
+)
+
+const (
+	errorColor   = "\033[1;31m%s\033[0m"
+	warningColor = "\033[0;33m%s\033[0m"
+	noticeColor  = "\033[1;36m%s\033[0m"
+)
+
+// IsTerminal return true if the file descriptor is terminal and the TERM
+// environment variable isn't a dumb one.
+func IsTerminal(f *os.File) bool {
+	if runtime.GOOS == "windows" {
+		return false
+	}
+
+	fd := f.Fd()
+	return os.Getenv("TERM") != "dumb" && (isatty.IsTerminal(fd) || isatty.IsCygwinTerminal(fd))
+}
+
+// Notice colorizes the string in a noticeable color.
+func Notice(s string) string {
+	return colorize(s, noticeColor)
+}
+
+// Error colorizes the string in a colour that grabs attention.
+func Error(s string) string {
+	return colorize(s, errorColor)
+}
+
+// Warning colorizes the string in a colour that warns.
+func Warning(s string) string {
+	return colorize(s, warningColor)
+}
+
+// colorize s in color.
+func colorize(s, color string) string {
+	s = fmt.Sprintf(color, doublePercent(s))
+	return singlePercent(s)
+}
+
+func doublePercent(str string) string {
+	return strings.Replace(str, "%", "%%", -1)
+}
+
+func singlePercent(str string) string {
+	return strings.Replace(str, "%%", "%", -1)
+}

common/text/position.go

@@ -0,0 +1,99 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package text
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/gohugoio/hugo/common/terminal"
+)
+
+// Positioner represents a thing that knows its position in a text file or stream,
+// typically an error.
+type Positioner interface {
+	Position() Position
+}
+
+// Position holds a source position in a text file or stream.
+type Position struct {
+	Filename     string // filename, if any
+	Offset       int    // byte offset, starting at 0. It's set to -1 if not provided.
+	LineNumber   int    // line number, starting at 1
+	ColumnNumber int    // column number, starting at 1 (character count per line)
+}
+
+func (pos Position) String() string {
+	if pos.Filename == "" {
+		pos.Filename = "<stream>"
+	}
+	return positionStringFormatfunc(pos)
+}
+
+// IsValid returns true if line number is > 0.
+func (pos Position) IsValid() bool {
+	return pos.LineNumber > 0
+}
+
+var positionStringFormatfunc func(p Position) string
+
+func createPositionStringFormatter(formatStr string) func(p Position) string {
+
+	if formatStr == "" {
+		formatStr = "\":file::line::col\""
+	}
+
+	var identifiers = []string{":file", ":line", ":col"}
+	var identifiersFound []string
+
+	for i := range formatStr {
+		for _, id := range identifiers {
+			if strings.HasPrefix(formatStr[i:], id) {
+				identifiersFound = append(identifiersFound, id)
+			}
+		}
+	}
+
+	replacer := strings.NewReplacer(":file", "%s", ":line", "%d", ":col", "%d")
+	format := replacer.Replace(formatStr)
+
+	f := func(pos Position) string {
+		args := make([]interface{}, len(identifiersFound))
+		for i, id := range identifiersFound {
+			switch id {
+			case ":file":
+				args[i] = pos.Filename
+			case ":line":
+				args[i] = pos.LineNumber
+			case ":col":
+				args[i] = pos.ColumnNumber
+			}
+		}
+
+		msg := fmt.Sprintf(format, args...)
+
+		if terminal.IsTerminal(os.Stdout) {
+			return terminal.Notice(msg)
+		}
+
+		return msg
+	}
+
+	return f
+}
+
+func init() {
+	positionStringFormatfunc = createPositionStringFormatter(os.Getenv("HUGO_FILE_LOG_FORMAT"))
+}

common/text/position_test.go

@@ -0,0 +1,33 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package text
+
+import (
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestPositionStringFormatter(t *testing.T) {
+	c := qt.New(t)
+
+	pos := Position{Filename: "/my/file.txt", LineNumber: 12, ColumnNumber: 13, Offset: 14}
+
+	c.Assert(createPositionStringFormatter(":file|:col|:line")(pos), qt.Equals, "/my/file.txt|13|12")
+	c.Assert(createPositionStringFormatter(":col|:file|:line")(pos), qt.Equals, "13|/my/file.txt|12")
+	c.Assert(createPositionStringFormatter("好::col")(pos), qt.Equals, "好:13")
+	c.Assert(createPositionStringFormatter("")(pos), qt.Equals, "\"/my/file.txt:12:13\"")
+	c.Assert(pos.String(), qt.Equals, "\"/my/file.txt:12:13\"")
+
+}

common/types/evictingqueue.go

@@ -52,6 +52,13 @@ func (q *EvictingStringQueue) Add(v string) {
 	q.mu.Unlock()
 }
 
+// Contains returns whether the queue contains v.
+func (q *EvictingStringQueue) Contains(v string) bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return q.set[v]
+}
+
 // Peek looks at the last element added to the queue.
 func (q *EvictingStringQueue) Peek() string {
 	q.mu.Lock()

common/types/evictingqueue_test.go

@@ -17,33 +17,36 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/stretchr/testify/require"
+	qt "github.com/frankban/quicktest"
 )
 
 func TestEvictingStringQueue(t *testing.T) {
-	assert := require.New(t)
+	c := qt.New(t)
 
 	queue := NewEvictingStringQueue(3)
 
-	assert.Equal("", queue.Peek())
+	c.Assert(queue.Peek(), qt.Equals, "")
 	queue.Add("a")
 	queue.Add("b")
 	queue.Add("a")
-	assert.Equal("b", queue.Peek())
+	c.Assert(queue.Peek(), qt.Equals, "b")
 	queue.Add("b")
-	assert.Equal("b", queue.Peek())
+	c.Assert(queue.Peek(), qt.Equals, "b")
 
 	queue.Add("a")
 	queue.Add("b")
 
-	assert.Equal([]string{"b", "a"}, queue.PeekAll())
-	assert.Equal("b", queue.Peek())
+	c.Assert(queue.Contains("a"), qt.Equals, true)
+	c.Assert(queue.Contains("foo"), qt.Equals, false)
+
+	c.Assert(queue.PeekAll(), qt.DeepEquals, []string{"b", "a"})
+	c.Assert(queue.Peek(), qt.Equals, "b")
 	queue.Add("c")
 	queue.Add("d")
 	// Overflowed, a should now be removed.
-	assert.Equal([]string{"d", "c", "b"}, queue.PeekAll())
-	assert.Len(queue.PeekAllSet(), 3)
-	assert.True(queue.PeekAllSet()["c"])
+	c.Assert(queue.PeekAll(), qt.DeepEquals, []string{"d", "c", "b"})
+	c.Assert(len(queue.PeekAllSet()), qt.Equals, 3)
+	c.Assert(queue.PeekAllSet()["c"], qt.Equals, true)
 }
 
 func TestEvictingStringQueueConcurrent(t *testing.T) {

common/types/types.go

@@ -1,4 +1,4 @@
-// Copyright 2017-present The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@ package types
 
 import (
 	"fmt"
+	"reflect"
 
 	"github.com/spf13/cast"
 )
@@ -41,6 +42,8 @@ func (k KeyValues) String() string {
 	return fmt.Sprintf("%v: %v", k.Key, k.Values)
 }
 
+// NewKeyValuesStrings takes a given key and slice of values and returns a new
+// KeyValues struct.
 func NewKeyValuesStrings(key string, values ...string) KeyValues {
 	iv := make([]interface{}, len(values))
 	for i := 0; i < len(values); i++ {
@@ -48,3 +51,30 @@ func NewKeyValuesStrings(key string, values ...string) KeyValues {
 	}
 	return KeyValues{Key: key, Values: iv}
 }
+
+// Zeroer, as implemented by time.Time, will be used by the truth template
+// funcs in Hugo (if, with, not, and, or).
+type Zeroer interface {
+	IsZero() bool
+}
+
+// IsNil reports whether v is nil.
+func IsNil(v interface{}) bool {
+	if v == nil {
+		return true
+	}
+
+	value := reflect.ValueOf(v)
+	switch value.Kind() {
+	case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
+		return value.IsNil()
+	}
+
+	return false
+}
+
+// DevMarker is a marker interface for types that should only be used during
+// development.
+type DevMarker interface {
+	DevOnly()
+}

common/types/types_test.go

@@ -16,14 +16,14 @@ package types
 import (
 	"testing"
 
-	"github.com/stretchr/testify/require"
+	qt "github.com/frankban/quicktest"
 )
 
 func TestKeyValues(t *testing.T) {
-	assert := require.New(t)
+	c := qt.New(t)
 
 	kv := NewKeyValuesStrings("key", "a1", "a2")
 
-	assert.Equal("key", kv.KeyString())
-	assert.Equal([]interface{}{"a1", "a2"}, kv.Values)
+	c.Assert(kv.KeyString(), qt.Equals, "key")
+	c.Assert(kv.Values, qt.DeepEquals, []interface{}{"a1", "a2"})
 }

common/urls/ref.go

@@ -0,0 +1,22 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package urls
+
+// RefLinker is implemented by those who support reference linking.
+// args must contain a path, but can also point to the target
+// language or output format.
+type RefLinker interface {
+	Ref(args map[string]interface{}) (string, error)
+	RelRef(args map[string]interface{}) (string, error)
+}

compare/compare.go

@@ -1,4 +1,4 @@
-// Copyright 2017-present The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,6 +20,12 @@ type Eqer interface {
 	Eq(other interface{}) bool
 }
 
+// ProbablyEqer is an equal check that may return false positives, but never
+// a false negative.
+type ProbablyEqer interface {
+	ProbablyEq(other interface{}) bool
+}
+
 // Comparer can be used to compare two values.
 // This will be used when using the le, ge etc. operators in the templates.
 // Compare returns -1 if the given version is less than, 0 if equal and 1 if greater than

compare/compare_strings.go

@@ -0,0 +1,113 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compare
+
+import (
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+// Strings returns an integer comparing two strings lexicographically.
+func Strings(s, t string) int {
+	c := compareFold(s, t)
+
+	if c == 0 {
+		// "B" and "b" would be the same so we need a tiebreaker.
+		return strings.Compare(s, t)
+	}
+
+	return c
+}
+
+// This function is derived from strings.EqualFold in Go's stdlib.
+// https://github.com/golang/go/blob/ad4a58e31501bce5de2aad90a620eaecdc1eecb8/src/strings/strings.go#L893
+func compareFold(s, t string) int {
+	for s != "" && t != "" {
+		var sr, tr rune
+		if s[0] < utf8.RuneSelf {
+			sr, s = rune(s[0]), s[1:]
+		} else {
+			r, size := utf8.DecodeRuneInString(s)
+			sr, s = r, s[size:]
+		}
+		if t[0] < utf8.RuneSelf {
+			tr, t = rune(t[0]), t[1:]
+		} else {
+			r, size := utf8.DecodeRuneInString(t)
+			tr, t = r, t[size:]
+		}
+
+		if tr == sr {
+			continue
+		}
+
+		c := 1
+		if tr < sr {
+			tr, sr = sr, tr
+			c = -c
+		}
+
+		//  ASCII only.
+		if tr < utf8.RuneSelf {
+			if sr >= 'A' && sr <= 'Z' {
+				if tr <= 'Z' {
+					// Same case.
+					return -c
+				}
+
+				diff := tr - (sr + 'a' - 'A')
+
+				if diff == 0 {
+					continue
+				}
+
+				if diff < 0 {
+					return c
+				}
+
+				if diff > 0 {
+					return -c
+				}
+			}
+		}
+
+		// Unicode.
+		r := unicode.SimpleFold(sr)
+		for r != sr && r < tr {
+			r = unicode.SimpleFold(r)
+		}
+
+		if r == tr {
+			continue
+		}
+
+		return -c
+	}
+
+	if s == "" && t == "" {
+		return 0
+	}
+
+	if s == "" {
+		return -1
+	}
+
+	return 1
+}
+
+// LessStrings returns whether s is less than t lexicographically.
+func LessStrings(s, t string) bool {
+	return Strings(s, t) < 0
+}

compare/compare_strings_test.go

@@ -0,0 +1,65 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compare
+
+import (
+	"sort"
+	"strings"
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestCompare(t *testing.T) {
+	c := qt.New(t)
+	for _, test := range []struct {
+		a string
+		b string
+	}{
+		{"a", "a"},
+		{"A", "a"},
+		{"Ab", "Ac"},
+		{"az", "Za"},
+		{"C", "D"},
+		{"B", "a"},
+		{"C", ""},
+		{"", ""},
+		{"αβδC", "ΑΒΔD"},
+		{"αβδC", "ΑΒΔ"},
+		{"αβδ", "ΑΒΔD"},
+		{"αβδ", "ΑΒΔ"},
+		{"β", "δ"},
+		{"好", strings.ToLower("好")},
+	} {
+
+		expect := strings.Compare(strings.ToLower(test.a), strings.ToLower(test.b))
+		got := compareFold(test.a, test.b)
+
+		c.Assert(got, qt.Equals, expect)
+
+	}
+}
+
+func TestLexicographicSort(t *testing.T) {
+	c := qt.New(t)
+
+	s := []string{"b", "Bz", "ba", "A", "Ba", "ba"}
+
+	sort.Slice(s, func(i, j int) bool {
+		return LessStrings(s[i], s[j])
+	})
+
+	c.Assert(s, qt.DeepEquals, []string{"A", "b", "Ba", "ba", "ba", "Bz"})
+
+}

config/configLoader.go

@@ -0,0 +1,125 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/gohugoio/hugo/common/maps"
+	"github.com/gohugoio/hugo/parser/metadecoders"
+	"github.com/spf13/afero"
+	"github.com/spf13/viper"
+)
+
+var (
+	ValidConfigFileExtensions                    = []string{"toml", "yaml", "yml", "json"}
+	validConfigFileExtensionsMap map[string]bool = make(map[string]bool)
+)
+
+func init() {
+	for _, ext := range ValidConfigFileExtensions {
+		validConfigFileExtensionsMap[ext] = true
+	}
+}
+
+// IsValidConfigFilename returns whether filename is one of the supported
+// config formats in Hugo.
+func IsValidConfigFilename(filename string) bool {
+	ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(filename), "."))
+	return validConfigFileExtensionsMap[ext]
+}
+
+// FromConfigString creates a config from the given YAML, JSON or TOML config. This is useful in tests.
+func FromConfigString(config, configType string) (Provider, error) {
+	v := newViper()
+	m, err := readConfig(metadecoders.FormatFromString(configType), []byte(config))
+	if err != nil {
+		return nil, err
+	}
+
+	v.MergeConfigMap(m)
+
+	return v, nil
+}
+
+// FromFile loads the configuration from the given filename.
+func FromFile(fs afero.Fs, filename string) (Provider, error) {
+	m, err := loadConfigFromFile(fs, filename)
+	if err != nil {
+		return nil, err
+	}
+
+	v := newViper()
+
+	err = v.MergeConfigMap(m)
+	if err != nil {
+		return nil, err
+	}
+
+	return v, nil
+}
+
+// FromFileToMap is the same as FromFile, but it returns the config values
+// as a simple map.
+func FromFileToMap(fs afero.Fs, filename string) (map[string]interface{}, error) {
+	return loadConfigFromFile(fs, filename)
+}
+
+func readConfig(format metadecoders.Format, data []byte) (map[string]interface{}, error) {
+	m, err := metadecoders.Default.UnmarshalToMap(data, format)
+	if err != nil {
+		return nil, err
+	}
+
+	RenameKeys(m)
+
+	return m, nil
+
+}
+
+func loadConfigFromFile(fs afero.Fs, filename string) (map[string]interface{}, error) {
+	m, err := metadecoders.Default.UnmarshalFileToMap(fs, filename)
+	if err != nil {
+		return nil, err
+	}
+	RenameKeys(m)
+	return m, nil
+}
+
+var keyAliases maps.KeyRenamer
+
+func init() {
+	var err error
+	keyAliases, err = maps.NewKeyRenamer(
+		// Before 0.53 we used singular for "menu".
+		"{menu,languages/*/menu}", "menus",
+	)
+
+	if err != nil {
+		panic(err)
+	}
+}
+
+// RenameKeys renames config keys in m recursively according to a global Hugo
+// alias definition.
+func RenameKeys(m map[string]interface{}) {
+	keyAliases.Rename(m)
+}
+
+func newViper() *viper.Viper {
+	v := viper.New()
+
+	return v
+}