releaser: Add release notes to /docs for release of 0.52

[ci skip]

.circleci/config.yml

@@ -1,8 +1,9 @@
 defaults: &defaults
-  working_directory: /go/src/github.com/gohugoio
   docker:
-      - image: bepsays/ci-goreleaser:0.34.2-6
-    
+      - image: bepsays/ci-goreleaser:1.11-2
+  environment:
+    CGO_ENABLED: "0"
+
 version: 2
 jobs:
   build:
@@ -12,11 +13,11 @@ jobs:
           path: hugo
       - run:
             command: |
-                go get -d github.com/magefile/mage/...
                 git clone git@github.com:gohugoio/hugoDocs.git
                 cd hugo
-                mage vendor
-                mage check
+                go mod download
+                sleep 5
+                go test -p 1 ./...
       - persist_to_workspace:
           root: .
           paths: .
@@ -24,7 +25,7 @@ jobs:
     <<: *defaults
     steps:
       - attach_workspace:
-          at: /go/src/github.com/gohugoio
+          at: /root/project
       - run:
             command: |
                     cd hugo

.dockerignore

@@ -0,0 +1,8 @@
+*.md
+*.log
+*.txt
+.git
+.github
+.circleci
+docs
+examples

.gitattributes

@@ -0,0 +1,8 @@
+# Text files have auto line endings
+* text=auto
+
+# Go source files always have LF line endings
+*.go text eol=lf
+
+# SVG files should not be modified
+*.svg -text

.github/SUPPORT.md

@@ -0,0 +1,3 @@
+### Asking Support Questions
+
+We have an active [discussion forum](https://discourse.gohugo.io) where users and developers can ask questions. Please don't use the GitHub issue tracker to ask questions.

.github/stale.yml

@@ -1,7 +1,7 @@
 # Number of days of inactivity before an issue becomes stale
 daysUntilStale: 120
 # Number of days of inactivity before a stale issue is closed
-daysUntilClose: 21
+daysUntilClose: 30
 # Issues with these labels will never be considered stale
 exemptLabels:
   - Keep
@@ -20,4 +20,4 @@ markComment: >
   This issue will automatically be closed in the near future if no further activity occurs. Thank you for all your contributions.
   
 # Comment to post when closing a stale issue. Set to `false` to disable
-closeComment: false
+closeComment: false

.gitignore

@@ -12,7 +12,13 @@ cover.out
 *~
 vendor/*/
 *.bench
+*.debug
 coverage*.out
 
+dock.sh
+
 GoBuilds
 dist
+
+
+vendor

.travis.yml

@@ -1,25 +1,44 @@
 language: go
-sudo: required
-dist: precise
+sudo: false
+dist: trusty
+env:
+  global:
+  - HUGO_BUILD_TAGS="extended"
+git:
+  depth: false
 go:
-  - 1.8.5
-  - 1.9.2
+  - 1.11
   - tip
 os:
   - linux
   - osx
+  - windows
 matrix:
   allow_failures:
     - go: tip
   fast_finish: true
+  exclude:
+  - os: windows
+    go: tip
+
 install:
+  - mkdir -p $HOME/src
+  - mv $HOME/gopath/src/github.com/gohugoio/hugo $HOME/src
+  - export TRAVIS_BUILD_DIR=$HOME/src/hugo
+  - cd $HOME/src/hugo
   - go get github.com/magefile/mage
-  - mage vendor
 script:
-  - mage hugoRace
+  - go mod download
+  - mage -v test
   - mage -v check
+  - mage -v hugo
   - ./hugo -s docs/
   - ./hugo --renderToMemory -s docs/
+  - df -h
+
 before_install:
-  # gem install must be run with sudo on OSX
-  - sudo gem install asciidoctor | gem install asciidoctor
+  - df -h
+    # https://travis-ci.community/t/go-cant-find-gcc-with-go1-11-1-on-windows/293/5
+  - if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then choco install mingw -y; export PATH=/c/tools/mingw64/bin:"$PATH"; fi
+  - gem install asciidoctor
+  - type asciidoctor

CONTRIBUTING.md

@@ -9,7 +9,7 @@ The Hugo community and maintainers are [very active](https://github.com/gohugoio
 
 *Note that this repository only contains the actual source code of Hugo. For **only** documentation-related pull requests / issues please refer to the [hugoDocs](https://github.com/gohugoio/hugoDocs) repository.*
 
-*Pull requests that contain changes on the code base **and** related documentation, e.g. for a new feature, shall remain a single, atomic one.*
+*Changes to the codebase **and** related documentation, e.g. for a new feature, should still use a single pull request.*
 
 ## Table of Contents
 
@@ -18,11 +18,8 @@ The Hugo community and maintainers are [very active](https://github.com/gohugoio
 * [Submitting Patches](#submitting-patches)
   * [Code Contribution Guidelines](#code-contribution-guidelines)
   * [Git Commit Message Guidelines](#git-commit-message-guidelines)
-  * [Vendored Dependencies](#vendored-dependencies)
   * [Fetching the Sources From GitHub](#fetching-the-sources-from-github)
-  * [Using Git Remotes](#using-git-remotes)
-  * [Build Hugo with Your Changes](#build-hugo-with-your-changes)
-  * [Updating the Hugo Sources](#updating-the-hugo-sources)
+  * [Building Hugo with Your Changes](#building-hugo-with-your-changes)
 
 ## Asking Support Questions
 
@@ -32,22 +29,38 @@ Please don't use the GitHub issue tracker to ask questions.
 ## Reporting Issues
 
 If you believe you have found a defect in Hugo or its documentation, use
-the GitHub [issue tracker](https://github.com/gohugoio/hugo/issues) to report the problem to the Hugo maintainers.
-If you're not sure if it's a bug or not, start by asking in the [discussion forum](https://discourse.gohugo.io).
-When reporting the issue, please provide the version of Hugo in use (`hugo version`) and your operating system.
+the GitHub [issue tracker](https://github.com/gohugoio/hugo/issues) to report
+the problem to the Hugo maintainers. If you're not sure if it's a bug or not,
+start by asking in the [discussion forum](https://discourse.gohugo.io).
+When reporting the issue, please provide the version of Hugo in use (`hugo
+version`) and your operating system.
+
+## Code Contribution
+
+Hugo has become a fully featured static site generator, so any new functionality must:
+
+* be useful to many.
+* fit naturally into _what Hugo does best._
+* strive not to break existing sites.
+* close or update an open [Hugo issue](https://github.com/gohugoio/hugo/issues)
+
+If it is of some complexity, the contributor is expected to maintain and support the new future (answer questions on the forum, fix any bugs etc.).
+
+It is recommended to open up a discussion on the [Hugo Forum](https://discourse.gohugo.io/) to get feedback on your idea before you begin. If you are submitting a complex feature, create a small design proposal on the [Hugo issue tracker](https://github.com/gohugoio/hugo/issues) before you start.
+
+
+**Bug fixes are, of course, always welcome.**
+
+
 
 ## Submitting Patches
 
-The Hugo project welcomes all contributors and contributions regardless of skill or experience level.
-If you are interested in helping with the project, we will help you with your contribution.
-Hugo is a very active project with many contributions happening daily.
-Because we want to create the best possible product for our users and the best contribution experience for our developers,
-we have a set of guidelines which ensure that all contributions are acceptable.
-The guidelines are not intended as a filter or barrier to participation.
-If you are unfamiliar with the contribution process, the Hugo team will help you and teach you how to bring your contribution in accordance with the guidelines.
+The Hugo project welcomes all contributors and contributions regardless of skill or experience level. If you are interested in helping with the project, we will help you with your contribution.
 
 ### Code Contribution Guidelines
 
+Because we want to create the best possible product for our users and the best contribution experience for our developers, we have a set of guidelines which ensure that all contributions are acceptable. The guidelines are not intended as a filter or barrier to participation. If you are unfamiliar with the contribution process, the Hugo team will help you and teach you how to bring your contribution in accordance with the guidelines.
+
 To make the contribution process as seamless as possible, we ask for the following:
 
 * Go ahead and fork the project and make your changes.  We encourage pull requests to allow for review and discussion of code changes.
@@ -57,7 +70,7 @@ To make the contribution process as seamless as possible, we ask for the followi
     * Run `go fmt`.
     * Add documentation if you are adding new features or changing functionality.  The docs site lives in `/docs`.
     * Squash your commits into a single commit. `git rebase -i`. It’s okay to force update your pull request with `git push -f`.
-    * Ensure that `mage check` succeeds. [Travis CI](https://travis-ci.org/gohugoio/hugo) (Linux and macOS) and [AppVeyor](https://ci.appveyor.com/project/gohugoio/hugo/branch/master) (Windows) will fail the build if `mage check` fails.
+    * Ensure that `mage check` succeeds. [Travis CI](https://travis-ci.org/gohugoio/hugo) (Windows, Linux and macOS) will fail the build if `mage check` fails.
     * Follow the **Git Commit Message Guidelines** below.
 
 ### Git Commit Message Guidelines
@@ -69,11 +82,11 @@ the most important part being that each commit message should have a title/subje
 Also, if your commit references one or more GitHub issues, always end your commit message body with *See #1234* or *Fixes #1234*.
 Replace *1234* with the GitHub issue ID. The last example will close the issue when the commit is merged into *master*.
 
-Sometimes it makes sense to prefix the commit message with the packagename (or docs folder) all lowercased ending with a colon.
+Sometimes it makes sense to prefix the commit message with the package name (or docs folder) all lowercased ending with a colon.
 That is fine, but the rest of the rules above apply.
 So it is "tpl: Add emojify template func", not "tpl: add emojify template func.", and "docs: Document emoji", not "doc: document emoji."
 
-Please consider to use a short and descriptive branch name, e.g. **NOT** "patch-1". It's very common but creates a naming conflict each time when a submission is pulled for a review.
+Please use a short and descriptive branch name, e.g. **NOT** "patch-1". It's very common but creates a naming conflict each time when a submission is pulled for a review.
 
 An example:
 
@@ -90,30 +103,25 @@ Fixes #1949
 
 ###  Fetching the Sources From GitHub
 
-Due to the way Go handles package imports, the best approach for working on a
-Hugo fork is to use Git Remotes.  Here's a simple walk-through for getting
-started:
+Since Hugo 0.48, Hugo uses the Go Modules support built into Go 1.11 to build. The easiest is to clone Hugo in a directory outside of `GOPATH`, as in the following example:
 
-1. Get the Hugo source:
+```bash
+mkdir $HOME/src
+cd $HOME/src
+git clone https://github.com/gohugoio/hugo.git
+cd hugo
+go install
+```
 
-    ```bash
-    go get -u -v -d github.com/gohugoio/hugo
-    ```
+>Note: Some Go tools may not be fully updated to support Go Modules yet. One example would be LiteIDE. Follow [this workaround](https://github.com/visualfc/liteide/issues/986#issuecomment-428117702) for how to continue to work with Hugo below `GOPATH`.
 
-1. Install Mage:
+For some convenient build and test targets, you also will want to install Mage:
 
-    ```bash
-    go get github.com/magefile/mage
-    ```
+```bash
+go get github.com/magefile/mage
+```
 
-1. Change to the Hugo source directory and fetch the dependencies:
-
-    ```bash
-    cd $HOME/go/src/github.com/gohugoio/hugo
-    mage vendor
-    ```
-
-    Note that Hugo uses [Go Dep](https://github.com/golang/dep) to vendor dependencies, rather than a a simple `go get`. We don't commit the vendored packages themselves to the Hugo git repository. The call to `mage vendor` takes care of all this for you.
+Now, to make a change to Hugo's source:
 
 1. Create a new branch for your changes (the branch name is arbitrary):
 
@@ -151,7 +159,7 @@ Hugo uses [mage](https://github.com/magefile/mage) to sync vendor dependencies,
 cd $HOME/go/src/github.com/gohugoio/hugo
 ```
 
-To build Hugo: 
+To build Hugo:
 
 ```bash
 mage hugo
@@ -176,14 +184,8 @@ To list all available commands along with descriptions:
 mage -l
 ```
 
-### Updating the Hugo Sources
-
-If you want to stay in sync with the Hugo repository, you can easily pull down
-the source changes, but you'll need to keep the vendored packages up-to-date as
-well.
+**Note:** From Hugo 0.43 we have added a build tag, `extended` that adds **SCSS support**. This needs a C compiler installed to build. You can enable this when building by:
 
 ```bash
-git pull
-mage vendor
-```
-
+HUGO_BUILD_TAGS=extended mage install
+````

Dockerfile

@@ -1,23 +1,26 @@
-FROM golang:1.9.0-alpine3.6 AS build
+# GitHub:       https://github.com/gohugoio
+# Twitter:      https://twitter.com/gohugoio
+# Website:      https://gohugo.io/
 
-RUN apk add --no-cache --virtual git musl-dev
-RUN go get github.com/golang/dep/cmd/dep
+FROM golang:1.11-alpine3.7 AS build
+
+ENV CGO_ENABLED=0
+ENV GOOS=linux
+ENV GO111MODULE=on
 
 WORKDIR /go/src/github.com/gohugoio/hugo
-RUN dep ensure
-ADD . /go/src/github.com/gohugoio/hugo/
+RUN apk add --no-cache \
+    git \
+    musl-dev
+COPY . /go/src/github.com/gohugoio/hugo/
 RUN go install -ldflags '-s -w'
 
-FROM alpine:3.6
-RUN \
-  adduser -h /site -s /sbin/nologin -u 1000 -D hugo && \
-  apk add --no-cache \
-    dumb-init
-COPY --from=build /go/bin/hugo /bin/hugo
-USER    hugo
+# ---
+
+FROM scratch
+COPY --from=build /go/bin/hugo /hugo
 WORKDIR /site
 VOLUME  /site
 EXPOSE  1313
-
-ENTRYPOINT ["/usr/bin/dumb-init", "--", "hugo"]
+ENTRYPOINT [ "/hugo" ]
 CMD [ "--help" ]

Gopkg.lock

@@ -1,386 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  branch = "master"
-  name = "github.com/BurntSushi/toml"
-  packages = ["."]
-  revision = "a368813c5e648fee92e5f6c30e3944ff9d5e8895"
-
-[[projects]]
-  name = "github.com/PuerkitoBio/purell"
-  packages = ["."]
-  revision = "0bcb03f4b4d0a9428594752bd2a3b9aa0a9d4bd4"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/PuerkitoBio/urlesc"
-  packages = ["."]
-  revision = "de5bf2ad457846296e2031421a34e2568e304e35"
-
-[[projects]]
-  name = "github.com/alecthomas/chroma"
-  packages = [
-    ".",
-    "formatters",
-    "formatters/html",
-    "lexers",
-    "styles"
-  ]
-  revision = "e0f32fbcb59bc99c521910291b7ba8ecdb17ca34"
-  version = "v0.2.1"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/bep/gitmap"
-  packages = ["."]
-  revision = "de8030ebafb76c6e84d50ee6d143382637c00598"
-
-[[projects]]
-  name = "github.com/chaseadamsio/goorgeous"
-  packages = ["."]
-  revision = "dcf1ef873b8987bf12596fe6951c48347986eb2f"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/cpuguy83/go-md2man"
-  packages = ["md2man"]
-  revision = "1d903dcb749992f3741d744c0f8376b4bd7eb3e1"
-  version = "v1.0.7"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/danwakefield/fnmatch"
-  packages = ["."]
-  revision = "cbb64ac3d964b81592e64f957ad53df015803288"
-
-[[projects]]
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/disintegration/imaging"
-  packages = ["."]
-  revision = "dd50a3ee9985ccd313a2f03c398fcaedc96dc707"
-  version = "v1.2.4"
-
-[[projects]]
-  name = "github.com/dlclark/regexp2"
-  packages = [
-    ".",
-    "syntax"
-  ]
-  revision = "487489b64fb796de2e55f4e8a4ad1e145f80e957"
-  version = "v1.1.6"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/eknkc/amber"
-  packages = [
-    ".",
-    "parser"
-  ]
-  revision = "cdade1c073850f4ffc70a829e31235ea6892853b"
-
-[[projects]]
-  name = "github.com/fortytw2/leaktest"
-  packages = ["."]
-  revision = "7dad53304f9614c1c365755c1176a8e876fee3e8"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/fsnotify/fsnotify"
-  packages = ["."]
-  revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
-  version = "v1.4.7"
-
-[[projects]]
-  name = "github.com/gobwas/glob"
-  packages = [
-    ".",
-    "compiler",
-    "match",
-    "syntax",
-    "syntax/ast",
-    "syntax/lexer",
-    "util/runes",
-    "util/strings"
-  ]
-  revision = "bea32b9cd2d6f55753d94a28e959b13f0244797a"
-  version = "v0.2.2"
-
-[[projects]]
-  name = "github.com/gorilla/websocket"
-  packages = ["."]
-  revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/go-immutable-radix"
-  packages = ["."]
-  revision = "8aac2701530899b64bdea735a1de8da899815220"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/golang-lru"
-  packages = ["simplelru"]
-  revision = "0a025b7e63adc15a622f29b0b2c4c3848243bbf6"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/hcl"
-  packages = [
-    ".",
-    "hcl/ast",
-    "hcl/parser",
-    "hcl/scanner",
-    "hcl/strconv",
-    "hcl/token",
-    "json/parser",
-    "json/scanner",
-    "json/token"
-  ]
-  revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8"
-
-[[projects]]
-  name = "github.com/inconshreveable/mousetrap"
-  packages = ["."]
-  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
-  version = "v1.0"
-
-[[projects]]
-  name = "github.com/jdkato/prose"
-  packages = [
-    "internal/util",
-    "transform"
-  ]
-  revision = "20d3663d4bc9dd10d75abcde9d92e04b4861c674"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/kyokomi/emoji"
-  packages = ["."]
-  revision = "7e06b236c489543f53868841f188a294e3383eab"
-  version = "v1.5"
-
-[[projects]]
-  name = "github.com/magefile/mage"
-  packages = [
-    "mg",
-    "sh"
-  ]
-  revision = "2f974307b636f59c13b88704cf350a4772fef271"
-  version = "v1.0.2"
-
-[[projects]]
-  name = "github.com/magiconair/properties"
-  packages = ["."]
-  revision = "d419a98cdbed11a922bf76f257b7c4be79b50e73"
-  version = "v1.7.4"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/markbates/inflect"
-  packages = ["."]
-  revision = "a12c3aec81a6a938bf584a4bac567afed9256586"
-
-[[projects]]
-  name = "github.com/mattn/go-runewidth"
-  packages = ["."]
-  revision = "9e777a8366cce605130a531d2cd6363d07ad7317"
-  version = "v0.0.2"
-
-[[projects]]
-  name = "github.com/miekg/mmark"
-  packages = ["."]
-  revision = "fd2f6c1403b37925bd7fe13af05853b8ae58ee5f"
-  version = "v1.3.6"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/mitchellh/mapstructure"
-  packages = ["."]
-  revision = "b4575eea38cca1123ec2dc90c26529b5c5acfcff"
-
-[[projects]]
-  name = "github.com/nicksnyder/go-i18n"
-  packages = [
-    "i18n/bundle",
-    "i18n/language",
-    "i18n/translation"
-  ]
-  revision = "0dc1626d56435e9d605a29875701721c54bc9bbd"
-  version = "v1.10.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/olekukonko/tablewriter"
-  packages = ["."]
-  revision = "96aac992fc8b1a4c83841a6c3e7178d20d989625"
-
-[[projects]]
-  name = "github.com/pelletier/go-toml"
-  packages = ["."]
-  revision = "16398bac157da96aa88f98a2df640c7f32af1da2"
-  version = "v1.0.1"
-
-[[projects]]
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/russross/blackfriday"
-  packages = ["."]
-  revision = "4048872b16cc0fc2c5fd9eacf0ed2c2fedaa0c8c"
-  version = "v1.5"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/shurcooL/sanitized_anchor_name"
-  packages = ["."]
-  revision = "86672fcb3f950f35f2e675df2240550f2a50762f"
-
-[[projects]]
-  name = "github.com/spf13/afero"
-  packages = [
-    ".",
-    "mem"
-  ]
-  revision = "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c"
-  version = "v1.0.2"
-
-[[projects]]
-  name = "github.com/spf13/cast"
-  packages = ["."]
-  revision = "acbeb36b902d72a7a4c18e8f3241075e7ab763e4"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/cobra"
-  packages = [
-    ".",
-    "doc"
-  ]
-  revision = "b95ab734e27d33e0d8fbabf71ca990568d4e2020"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/fsync"
-  packages = ["."]
-  revision = "12a01e648f05a938100a26858d2d59a120307a18"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/jwalterweatherman"
-  packages = ["."]
-  revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/nitro"
-  packages = ["."]
-  revision = "24d7ef30a12da0bdc5e2eb370a79c659ddccf0e8"
-
-[[projects]]
-  name = "github.com/spf13/pflag"
-  packages = ["."]
-  revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/spf13/viper"
-  packages = ["."]
-  revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/stretchr/testify"
-  packages = [
-    "assert",
-    "require"
-  ]
-  revision = "b91bfb9ebec76498946beb6af7c0230c7cc7ba6c"
-  version = "v1.2.0"
-
-[[projects]]
-  name = "github.com/yosssi/ace"
-  packages = ["."]
-  revision = "ea038f4770b6746c3f8f84f14fa60d9fe1205b56"
-  version = "v0.0.5"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/image"
-  packages = [
-    "bmp",
-    "riff",
-    "tiff",
-    "tiff/lzw",
-    "vp8",
-    "vp8l",
-    "webp"
-  ]
-  revision = "12117c17ca67ffa1ce22e9409f3b0b0a93ac08c7"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "idna"
-  ]
-  revision = "5ccada7d0a7ba9aeb5d3aca8d3501b4c2a509fec"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sync"
-  packages = ["errgroup"]
-  revision = "fd80eb99c8f653c847d294a001bdf2a3a6f768f5"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  revision = "810d7000345868fc619eb81f46307107118f4ae1"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/text"
-  packages = [
-    "collate",
-    "collate/build",
-    "internal/colltab",
-    "internal/gen",
-    "internal/tag",
-    "internal/triegen",
-    "internal/ucd",
-    "language",
-    "secure/bidirule",
-    "transform",
-    "unicode/bidi",
-    "unicode/cldr",
-    "unicode/norm",
-    "unicode/rangetable",
-    "width"
-  ]
-  revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3"
-
-[[projects]]
-  branch = "v2"
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  revision = "d670f9405373e636a5a2765eea47fac0c9bc91a4"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "c80ffe69d34005d8d72a87cc491ce1d9c91272e4b7f8fbd22d4fda8973fa8556"
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,137 +0,0 @@
-
-
-[[constraint]]
-  name = "github.com/BurntSushi/toml"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/PuerkitoBio/purell"
-  version = "1.1.0"
-
-[[constraint]]
-  name = "github.com/alecthomas/chroma"
-  version = "^0.2.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/bep/gitmap"
-
-[[constraint]]
- name = "github.com/chaseadamsio/goorgeous"
- version = "^1.1.0"
-
-[[constraint]]
- name = "github.com/disintegration/imaging"
- version = "~v1.2.4"
-
-[[constraint]]
-  name = "github.com/magefile/mage"
-  version = "v1"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/eknkc/amber"
-
-[[constraint]]
-  name = "github.com/fortytw2/leaktest"
-  version = "1.1.0"
-
-[[constraint]]
-  name = "github.com/fsnotify/fsnotify"
-  version = "^1.4.0"
-
-[[constraint]]
-  name = "github.com/gorilla/websocket"
-  version = "1.2.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/hashicorp/go-immutable-radix"
-
-[[constraint]]
-  name = "github.com/jdkato/prose"
-  version = "1.1.0"
-
-[[constraint]]
-  name = "github.com/kyokomi/emoji"
-  version = "1.5.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/markbates/inflect"
-
-[[constraint]]
-  name = "github.com/miekg/mmark"
-  version = "^1.3.6"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/mitchellh/mapstructure"
-
-[[constraint]]
-  name = "github.com/nicksnyder/go-i18n"
-  version = "^1.10.0"
-
-[[constraint]]
-  name = "github.com/russross/blackfriday"
-  version = "^1.5.0"
-
-[[constraint]]
-  name = "github.com/spf13/afero"
-  version = "^1.0.1"
-
-[[constraint]]
-  name = "github.com/spf13/cast"
-  version = "^1.1.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/spf13/cobra"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/spf13/fsync"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/spf13/jwalterweatherman"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/spf13/nitro"
-
-[[constraint]]
-  name = "github.com/spf13/pflag"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/spf13/viper"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/stretchr/testify"
-  version = "1.1.4"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/olekukonko/tablewriter"
-
-[[constraint]]
-  name = "github.com/yosssi/ace"
-  version = "^0.0.5"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/image"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/text"
-
-[[constraint]]
-  branch = "v2"
-  name = "gopkg.in/yaml.v2"
-
-[[constraint]]
-  name = "github.com/gobwas/glob"
-  version = "0.2.2"

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

LICENSE.md

@@ -1,194 +0,0 @@
-Apache License
-==============
-
-_Version 2.0, January 2004_
-_&lt;<http://www.apache.org/licenses/>&gt;_
-
-### Terms and Conditions for use, reproduction, and distribution
-
-#### 1. Definitions
-
-“License” shall mean the terms and conditions for use, reproduction, and
-distribution as defined by Sections 1 through 9 of this document.
-
-“Licensor” shall mean the copyright owner or entity authorized by the copyright
-owner that is granting the License.
-
-“Legal Entity” shall mean the union of the acting entity and all other entities
-that control, are controlled by, or are under common control with that entity.
-For the purposes of this definition, “control” means **(i)** the power, direct or
-indirect, to cause the direction or management of such entity, whether by
-contract or otherwise, or **(ii)** ownership of fifty percent (50%) or more of the
-outstanding shares, or **(iii)** beneficial ownership of such entity.
-
-“You” (or “Your”) shall mean an individual or Legal Entity exercising
-permissions granted by this License.
-
-“Source” form shall mean the preferred form for making modifications, including
-but not limited to software source code, documentation source, and configuration
-files.
-
-“Object” form shall mean any form resulting from mechanical transformation or
-translation of a Source form, including but not limited to compiled object code,
-generated documentation, and conversions to other media types.
-
-“Work” shall mean the work of authorship, whether in Source or Object form, made
-available under the License, as indicated by a copyright notice that is included
-in or attached to the work (an example is provided in the Appendix below).
-
-“Derivative Works” shall mean any work, whether in Source or Object form, that
-is based on (or derived from) the Work and for which the editorial revisions,
-annotations, elaborations, or other modifications represent, as a whole, an
-original work of authorship. For the purposes of this License, Derivative Works
-shall not include works that remain separable from, or merely link (or bind by
-name) to the interfaces of, the Work and Derivative Works thereof.
-
-“Contribution” shall mean any work of authorship, including the original version
-of the Work and any modifications or additions to that Work or Derivative Works
-thereof, that is intentionally submitted to Licensor for inclusion in the Work
-by the copyright owner or by an individual or Legal Entity authorized to submit
-on behalf of the copyright owner. For the purposes of this definition,
-“submitted” means any form of electronic, verbal, or written communication sent
-to the Licensor or its representatives, including but not limited to
-communication on electronic mailing lists, source code control systems, and
-issue tracking systems that are managed by, or on behalf of, the Licensor for
-the purpose of discussing and improving the Work, but excluding communication
-that is conspicuously marked or otherwise designated in writing by the copyright
-owner as “Not a Contribution.”
-
-“Contributor” shall mean Licensor and any individual or Legal Entity on behalf
-of whom a Contribution has been received by Licensor and subsequently
-incorporated within the Work.
-
-#### 2. Grant of Copyright License
-
-Subject to the terms and conditions of this License, each Contributor hereby
-grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
-irrevocable copyright license to reproduce, prepare Derivative Works of,
-publicly display, publicly perform, sublicense, and distribute the Work and such
-Derivative Works in Source or Object form.
-
-#### 3. Grant of Patent License
-
-Subject to the terms and conditions of this License, each Contributor hereby
-grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
-irrevocable (except as stated in this section) patent license to make, have
-made, use, offer to sell, sell, import, and otherwise transfer the Work, where
-such license applies only to those patent claims licensable by such Contributor
-that are necessarily infringed by their Contribution(s) alone or by combination
-of their Contribution(s) with the Work to which such Contribution(s) was
-submitted. If You institute patent litigation against any entity (including a
-cross-claim or counterclaim in a lawsuit) alleging that the Work or a
-Contribution incorporated within the Work constitutes direct or contributory
-patent infringement, then any patent licenses granted to You under this License
-for that Work shall terminate as of the date such litigation is filed.
-
-#### 4. Redistribution
-
-You may reproduce and distribute copies of the Work or Derivative Works thereof
-in any medium, with or without modifications, and in Source or Object form,
-provided that You meet the following conditions:
-
-* **(a)** You must give any other recipients of the Work or Derivative Works a copy of
-this License; and
-* **(b)** You must cause any modified files to carry prominent notices stating that You
-changed the files; and
-* **(c)** You must retain, in the Source form of any Derivative Works that You distribute,
-all copyright, patent, trademark, and attribution notices from the Source form
-of the Work, excluding those notices that do not pertain to any part of the
-Derivative Works; and
-* **(d)** If the Work includes a “NOTICE” text file as part of its distribution, then any
-Derivative Works that You distribute must include a readable copy of the
-attribution notices contained within such NOTICE file, excluding those notices
-that do not pertain to any part of the Derivative Works, in at least one of the
-following places: within a NOTICE text file distributed as part of the
-Derivative Works; within the Source form or documentation, if provided along
-with the Derivative Works; or, within a display generated by the Derivative
-Works, if and wherever such third-party notices normally appear. The contents of
-the NOTICE file are for informational purposes only and do not modify the
-License. You may add Your own attribution notices within Derivative Works that
-You distribute, alongside or as an addendum to the NOTICE text from the Work,
-provided that such additional attribution notices cannot be construed as
-modifying the License.
-
-You may add Your own copyright statement to Your modifications and may provide
-additional or different license terms and conditions for use, reproduction, or
-distribution of Your modifications, or for any such Derivative Works as a whole,
-provided Your use, reproduction, and distribution of the Work otherwise complies
-with the conditions stated in this License.
-
-#### 5. Submission of Contributions
-
-Unless You explicitly state otherwise, any Contribution intentionally submitted
-for inclusion in the Work by You to the Licensor shall be under the terms and
-conditions of this License, without any additional terms or conditions.
-Notwithstanding the above, nothing herein shall supersede or modify the terms of
-any separate license agreement you may have executed with Licensor regarding
-such Contributions.
-
-#### 6. Trademarks
-
-This License does not grant permission to use the trade names, trademarks,
-service marks, or product names of the Licensor, except as required for
-reasonable and customary use in describing the origin of the Work and
-reproducing the content of the NOTICE file.
-
-#### 7. Disclaimer of Warranty
-
-Unless required by applicable law or agreed to in writing, Licensor provides the
-Work (and each Contributor provides its Contributions) on an “AS IS” BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
-including, without limitation, any warranties or conditions of TITLE,
-NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
-solely responsible for determining the appropriateness of using or
-redistributing the Work and assume any risks associated with Your exercise of
-permissions under this License.
-
-#### 8. Limitation of Liability
-
-In no event and under no legal theory, whether in tort (including negligence),
-contract, or otherwise, unless required by applicable law (such as deliberate
-and grossly negligent acts) or agreed to in writing, shall any Contributor be
-liable to You for damages, including any direct, indirect, special, incidental,
-or consequential damages of any character arising as a result of this License or
-out of the use or inability to use the Work (including but not limited to
-damages for loss of goodwill, work stoppage, computer failure or malfunction, or
-any and all other commercial damages or losses), even if such Contributor has
-been advised of the possibility of such damages.
-
-#### 9. Accepting Warranty or Additional Liability
-
-While redistributing the Work or Derivative Works thereof, You may choose to
-offer, and charge a fee for, acceptance of support, warranty, indemnity, or
-other liability obligations and/or rights consistent with this License. However,
-in accepting such obligations, You may act only on Your own behalf and on Your
-sole responsibility, not on behalf of any other Contributor, and only if You
-agree to indemnify, defend, and hold each Contributor harmless for any liability
-incurred by, or claims asserted against, such Contributor by reason of your
-accepting any such warranty or additional liability.
-
-_END OF TERMS AND CONDITIONS_
-
-### APPENDIX: How to apply the Apache License to your work
-
-To apply the Apache License to your work, attach the following boilerplate
-notice, with the fields enclosed by brackets `[]` replaced with your own
-identifying information. (Don't include the brackets!) The text should be
-enclosed in the appropriate comment syntax for the file format. We also
-recommend that a file or class name and description of purpose be included on
-the same “printed page” as the copyright notice for easier identification within
-third-party archives.
-
-    Copyright [yyyy] [name of copyright owner]
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.

README.md

@@ -1,19 +1,18 @@
 ![Hugo](https://raw.githubusercontent.com/gohugoio/hugoDocs/master/static/img/hugo-logo.png)
 
-A Fast and Flexible Static Site Generator built with love by [spf13](http://spf13.com/) and [friends](https://github.com/gohugoio/hugo/graphs/contributors) in [Go][].
+A Fast and Flexible Static Site Generator built with love by [bep](https://github.com/bep), [spf13](http://spf13.com/) and [friends](https://github.com/gohugoio/hugo/graphs/contributors) in [Go][].
 
 [Website](https://gohugo.io) |
 [Forum](https://discourse.gohugo.io) |
-[Developer Chat (no support)](https://gitter.im/gohugoio/hugo) |
-[Documentation](https://gohugo.io/overview/introduction/) |
-[Installation Guide](https://gohugo.io/overview/installing/) |
+[Developer Chat (no support)](https://gitter.im/spf13/hugo) |
+[Documentation](https://gohugo.io/getting-started/) |
+[Installation Guide](https://gohugo.io/getting-started/installing/) |
 [Contribution Guide](CONTRIBUTING.md) |
-[Twitter](http://twitter.com/gohugoio)
+[Twitter](https://twitter.com/gohugoio)
 
 [![GoDoc](https://godoc.org/github.com/gohugoio/hugo?status.svg)](https://godoc.org/github.com/gohugoio/hugo)
-[![Linux and macOS Build Status](https://api.travis-ci.org/gohugoio/hugo.svg?branch=master&label=Linux+and+macOS+build "Linux and macOS Build Status")](https://travis-ci.org/gohugoio/hugo)
-[![Windows Build Status](https://ci.appveyor.com/api/projects/status/a5mr220vsd091kua?svg=true&label=Windows+build "Windows Build Status")](https://ci.appveyor.com/project/bep/hugo/branch/master)
-[![Dev chat at https://gitter.im/gohugoio/hugo](https://img.shields.io/badge/gitter-developer_chat-46bc99.svg)](https://gitter.im/spf13/hugo?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Linux and macOS Build Status](https://api.travis-ci.org/gohugoio/hugo.svg?branch=master&label=Windows+and+Linux+and+macOS+build "Windows, Linux and macOS Build Status")](https://travis-ci.org/gohugoio/hugo)
+[![Dev chat at https://gitter.im/spf13/hugo](https://img.shields.io/badge/gitter-developer_chat-46bc99.svg)](https://gitter.im/spf13/hugo?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Go Report Card](https://goreportcard.com/badge/github.com/gohugoio/hugo)](https://goreportcard.com/report/github.com/gohugoio/hugo)
 
 ## Overview
@@ -36,7 +35,7 @@ Currently, we provide pre-built Hugo binaries for Windows, Linux, FreeBSD, NetBS
 
 Hugo may also be compiled from source wherever the Go compiler tool chain can run, e.g. for other operating systems including DragonFly BSD, OpenBSD, Plan 9, and Solaris.
 
-**Complete documentation is available at [Hugo Documentation][].**
+**Complete documentation is available at [Hugo Documentation](https://gohugo.io/getting-started/).**
 
 ## Choose How to Install
 
@@ -50,19 +49,28 @@ Building the binaries is an easy task for an experienced `go` getter.
 
 ### Install Hugo as Your Site Generator (Binary Install)
 
-Use the [installation instructions in the Hugo documentation](https://gohugo.io/overview/installing/).
+Use the [installation instructions in the Hugo documentation](https://gohugo.io/getting-started/installing/).
 
 ### Build and Install the Binaries from Source (Advanced Install)
 
-Add Hugo and its package dependencies to your go `src` directory.
+#### Prerequisite Tools
 
-    go get -v github.com/gohugoio/hugo
+* [Git](https://git-scm.com/)
+* [Go (at least Go 1.11)](https://golang.org/dl/)
 
-Once the `get` completes, you should find your new `hugo` (or `hugo.exe`) executable sitting inside `$GOPATH/bin/`.
+#### Fetch from GitHub
 
-To update Hugo’s dependencies, use `go get` with the `-u` option.
+Since Hugo 0.48, Hugo uses the Go Modules support built into Go 1.11 to build. The easiest is to clone Hugo in a directory outside of `GOPATH`, as in the following example:
 
-    go get -u -v github.com/gohugoio/hugo
+```bash
+mkdir $HOME/src
+cd $HOME/src
+git clone https://github.com/gohugoio/hugo.git
+cd hugo
+go install
+```
+
+**If you are a Windows user, substitute the `$HOME` environment variable above with `%USERPROFILE%`.**
 	
 ## The Hugo Documentation
 
@@ -111,3 +119,71 @@ For a complete guide to contributing code to Hugo, see the [Contribution Guide](
 
 [Go]: https://golang.org/
 [Hugo Documentation]: https://gohugo.io/overview/introduction/
+
+## Dependencies
+
+Hugo stands on the shoulder of many great open source libraries, in lexical order:
+
+ | Dependency  | License |
+ | :------------- | :------------- |
+ | [github.com/BurntSushi/locker](https://github.com/BurntSushi/locker) | The Unlicense |
+ | [github.com/BurntSushi/toml](https://github.com/BurntSushi/toml) | MIT License |
+ | [github.com/PuerkitoBio/purell](https://github.com/PuerkitoBio/purell) | BSD 3-Clause "New" or "Revised" License |
+ | [github.com/PuerkitoBio/urlesc](https://github.com/PuerkitoBio/urlesc) | BSD 3-Clause "New" or "Revised" License |
+ | [github.com/alecthomas/chroma](https://github.com/alecthomas/chroma) |  MIT License |
+ | [github.com/bep/debounce](https://github.com/bep/debounce) |    MIT License |
+ | [github.com/bep/gitmap](https://github.com/bep/gitmap) |  MIT License |
+ | [github.com/bep/go-tocss](https://github.com/bep/go-tocss) | MIT License |
+ | [github.com/chaseadamsio/goorgeous](https://github.com/chaseadamsio/goorgeous) | MIT License |
+ | [github.com/cpuguy83/go-md2man](https://github.com/cpuguy83/go-md2man) | MIT License |
+ | [github.com/danwakefield/fnmatch](https://github.com/danwakefield/fnmatch) | BSD 2-Clause "Simplified" License |
+ | [github.com/disintegration/imaging](https://github.com/disintegration/imaging) |  MIT License |
+ | [github.com/dlclark/regexp2](https://github.com/dlclark/regexp2) | MIT License |
+ | [github.com/eknkc/amber](https://github.com/eknkc/amber) | MIT License |
+ | [github.com/fsnotify/fsnotify](https://github.com/fsnotify/fsnotify) |  BSD 3-Clause "New" or "Revised" License |
+ | [github.com/gobwas/glob](https://github.com/gobwas/glob) | MIT License |
+ | [github.com/gorilla/websocket](https://github.com/gorilla/websocket) |  BSD 2-Clause "Simplified" License |
+ | [github.com/hashicorp/go-immutable-radix](https://github.com/hashicorp/go-immutable-radix) | Mozilla Public License 2.0 |
+ | [github.com/hashicorp/golang-lru](https://github.com/hashicorp/golang-lru) | Mozilla Public License 2.0 |
+ | [github.com/hashicorp/hcl](https://github.com/hashicorp/hcl) |   Mozilla Public License 2.0 |
+ | [github.com/jdkato/prose](https://github.com/jdkato/prose) |    MIT License |
+ | [github.com/kyokomi/emoji](https://github.com/kyokomi/emoji) |   MIT License |
+ | [github.com/magiconair/properties](https://github.com/magiconair/properties) | BSD 2-Clause "Simplified" License |
+ | [github.com/markbates/inflect](https://github.com/markbates/inflect) |  MIT License |
+ | [github.com/mattn/go-isatty](https://github.com/mattn/go-isatty) | MIT License |
+ | [github.com/mattn/go-runewidth](https://github.com/mattn/go-runewidth) | MIT License |
+ | [github.com/miekg/mmark](https://github.com/miekg/mmark) | Simplified BSD License |
+ | [github.com/mitchellh/hashstructure](https://github.com/mitchellh/hashstructure) | MIT License |
+ | [github.com/mitchellh/mapstructure](https://github.com/mitchellh/mapstructure) | MIT License |
+ | [github.com/muesli/smartcrop](https://github.com/muesli/smartcrop) |   MIT License |
+ | [github.com/nicksnyder/go-i18n](https://github.com/nicksnyder/go-i18n) | MIT License |
+ | [github.com/olekukonko/tablewriter](https://github.com/olekukonko/tablewriter) | MIT License |
+ | [github.com/pelletier/go-toml](https://github.com/pelletier/go-toml) |  MIT License |
+ | [github.com/pkg/errors](https://github.com/pkg/errors) |  BSD 2-Clause "Simplified" License |
+ | [github.com/russross/blackfriday](https://github.com/russross/blackfriday)  | Simplified BSD License |
+ | [github.com/shurcooL/sanitized_anchor_name](https://github.com/shurcooL/sanitized_anchor_name) | MIT License |
+ | [github.com/spf13/afero](https://github.com/spf13/afero) | Apache License 2.0 |
+ | [github.com/spf13/cast](https://github.com/spf13/cast) |  MIT License |
+ | [github.com/spf13/cobra](https://github.com/spf13/cobra) | Apache License 2.0 |
+ | [github.com/spf13/fsync](https://github.com/spf13/fsync) | MIT License |
+ | [github.com/spf13/jwalterweatherman](https://github.com/spf13/jwalterweatherman) | MIT License |
+ | [github.com/spf13/nitro](https://github.com/spf13/nitro) | Apache License 2.0 |
+ | [github.com/spf13/pflag](https://github.com/spf13/pflag) | BSD 3-Clause "New" or "Revised" License |
+ | [github.com/spf13/viper](https://github.com/spf13/viper) | MIT License |
+ | [github.com/tdewolff/minify](https://github.com/tdewolff/minify) | MIT License |
+ | [github.com/tdewolff/parse](https://github.com/tdewolff/parse) | MIT License |
+ | [github.com/wellington/go-libsass](https://github.com/wellington/go-libsass) | Apache License 2.0 |
+ | [github.com/yosssi/ace](https://github.com/yosssi/ace) |  MIT License |
+ | [golang.org/x/image](https://golang.org/x/image) | BSD 3-Clause "New" or "Revised" License |
+ | [golang.org/x/net](https://golang.org/x/net) | BSD 3-Clause "New" or "Revised" License |
+ | [golang.org/x/sync](https://golang.org/x/sync) | BSD 3-Clause "New" or "Revised" License |
+ | [golang.org/x/sys](https://golang.org/x/sys) | BSD 3-Clause "New" or "Revised" License |
+ | [golang.org/x/text](https://golang.org/x/text) | BSD 3-Clause "New" or "Revised" License
+ | [gopkg.in/yaml.v2](https://gopkg.in/yaml.v2) |    Apache License 2.0 |
+
+  
+ 
+  
+ 
+ 
+ 

appveyor.yml

@@ -1,18 +0,0 @@
-init:
-  - set PATH=%PATH%;C:\MinGW\bin;%GOPATH%\bin
-  - go version
-  - go env
-
-# clones and cd's to path
-clone_folder: C:\GOPATH\src\github.com\gohugoio\hugo
-
-install:
-  - gem install asciidoctor
-  - pip install docutils
-  - go get github.com/magefile/mage
-
-build_script:
-  - mage vendor hugoRace
-  - mage -v check
-  - hugo -s docs/
-  - hugo --renderToMemory -s docs/

bepdock.sh

@@ -0,0 +1 @@
+docker run --rm --mount type=bind,source="$(pwd)",target=/hugo -w /hugo  -i -t bepsays/ci-goreleaser:1.11-2 /bin/bash

cache/filecache/filecache.go

@@ -0,0 +1,353 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gohugoio/hugo/common/hugio"
+
+	"github.com/gohugoio/hugo/helpers"
+
+	"github.com/BurntSushi/locker"
+	"github.com/spf13/afero"
+)
+
+const (
+	filecacheRootDirname = "filecache"
+)
+
+// Cache caches a set of files in a directory. This is usually a file on
+// disk, but since this is backed by an Afero file system, it can be anything.
+type Cache struct {
+	Fs afero.Fs
+
+	// Max age for items in this cache. Negative duration means forever,
+	// 0 is effectively turning this cache off.
+	maxAge time.Duration
+
+	nlocker *lockTracker
+}
+
+type lockTracker struct {
+	seenMu sync.RWMutex
+	seen   map[string]struct{}
+
+	*locker.Locker
+}
+
+// Lock tracks the ids in use. We use this information to do garbage collection
+// after a Hugo build.
+func (l *lockTracker) Lock(id string) {
+	l.seenMu.RLock()
+	if _, seen := l.seen[id]; !seen {
+		l.seenMu.RUnlock()
+		l.seenMu.Lock()
+		l.seen[id] = struct{}{}
+		l.seenMu.Unlock()
+	} else {
+		l.seenMu.RUnlock()
+	}
+
+	l.Locker.Lock(id)
+}
+
+// ItemInfo contains info about a cached file.
+type ItemInfo struct {
+	// This is the file's name relative to the cache's filesystem.
+	Name string
+}
+
+// NewCache creates a new file cache with the given filesystem and max age.
+func NewCache(fs afero.Fs, maxAge time.Duration) *Cache {
+	return &Cache{
+		Fs:      fs,
+		nlocker: &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})},
+		maxAge:  maxAge,
+	}
+}
+
+// lockedFile is a file with a lock that is released on Close.
+type lockedFile struct {
+	afero.File
+	unlock func()
+}
+
+func (l *lockedFile) Close() error {
+	defer l.unlock()
+	return l.File.Close()
+}
+
+// WriteCloser returns a transactional writer into the cache.
+// It's important that it's closed when done.
+func (c *Cache) WriteCloser(id string) (ItemInfo, io.WriteCloser, error) {
+	id = cleanID(id)
+	c.nlocker.Lock(id)
+
+	info := ItemInfo{Name: id}
+
+	f, err := helpers.OpenFileForWriting(c.Fs, id)
+	if err != nil {
+		c.nlocker.Unlock(id)
+		return info, nil, err
+	}
+
+	return info, &lockedFile{
+		File:   f,
+		unlock: func() { c.nlocker.Unlock(id) },
+	}, nil
+}
+
+// ReadOrCreate tries to lookup the file in cache.
+// If found, it is passed to read and then closed.
+// If not found a new file is created and passed to create, which should close
+// it when done.
+func (c *Cache) ReadOrCreate(id string,
+	read func(info ItemInfo, r io.Reader) error,
+	create func(info ItemInfo, w io.WriteCloser) error) (info ItemInfo, err error) {
+	id = cleanID(id)
+
+	c.nlocker.Lock(id)
+	defer c.nlocker.Unlock(id)
+
+	info = ItemInfo{Name: id}
+
+	if r := c.getOrRemove(id); r != nil {
+		err = read(info, r)
+		defer r.Close()
+		return
+	}
+
+	f, err := helpers.OpenFileForWriting(c.Fs, id)
+	if err != nil {
+		return
+	}
+
+	err = create(info, f)
+
+	return
+
+}
+
+// GetOrCreate tries to get the file with the given id from cache. If not found or expired, create will
+// be invoked and the result cached.
+// This method is protected by a named lock using the given id as identifier.
+func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (ItemInfo, io.ReadCloser, error) {
+	id = cleanID(id)
+
+	c.nlocker.Lock(id)
+	defer c.nlocker.Unlock(id)
+
+	info := ItemInfo{Name: id}
+
+	if r := c.getOrRemove(id); r != nil {
+		return info, r, nil
+	}
+
+	r, err := create()
+	if err != nil {
+		return info, nil, err
+	}
+
+	if c.maxAge == 0 {
+		// No caching.
+		return info, hugio.ToReadCloser(r), nil
+	}
+
+	var buff bytes.Buffer
+	return info,
+		hugio.ToReadCloser(&buff),
+		afero.WriteReader(c.Fs, id, io.TeeReader(r, &buff))
+}
+
+// GetOrCreateBytes is the same as GetOrCreate, but produces a byte slice.
+func (c *Cache) GetOrCreateBytes(id string, create func() ([]byte, error)) (ItemInfo, []byte, error) {
+	id = cleanID(id)
+
+	c.nlocker.Lock(id)
+	defer c.nlocker.Unlock(id)
+
+	info := ItemInfo{Name: id}
+
+	if r := c.getOrRemove(id); r != nil {
+		defer r.Close()
+		b, err := ioutil.ReadAll(r)
+		return info, b, err
+	}
+
+	b, err := create()
+	if err != nil {
+		return info, nil, err
+	}
+
+	if c.maxAge == 0 {
+		return info, b, nil
+	}
+
+	if err := afero.WriteReader(c.Fs, id, bytes.NewReader(b)); err != nil {
+		return info, nil, err
+	}
+	return info, b, nil
+
+}
+
+// GetBytes gets the file content with the given id from the cahce, nil if none found.
+func (c *Cache) GetBytes(id string) (ItemInfo, []byte, error) {
+	id = cleanID(id)
+
+	c.nlocker.Lock(id)
+	defer c.nlocker.Unlock(id)
+
+	info := ItemInfo{Name: id}
+
+	if r := c.getOrRemove(id); r != nil {
+		defer r.Close()
+		b, err := ioutil.ReadAll(r)
+		return info, b, err
+	}
+
+	return info, nil, nil
+}
+
+// Get gets the file with the given id from the cahce, nil if none found.
+func (c *Cache) Get(id string) (ItemInfo, io.ReadCloser, error) {
+	id = cleanID(id)
+
+	c.nlocker.Lock(id)
+	defer c.nlocker.Unlock(id)
+
+	info := ItemInfo{Name: id}
+
+	r := c.getOrRemove(id)
+
+	return info, r, nil
+}
+
+// getOrRemove gets the file with the given id. If it's expired, it will
+// be removed.
+func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser {
+	if c.maxAge == 0 {
+		// No caching.
+		return nil
+	}
+
+	if c.maxAge > 0 {
+		fi, err := c.Fs.Stat(id)
+		if err != nil {
+			return nil
+		}
+
+		if c.isExpired(fi.ModTime()) {
+			c.Fs.Remove(id)
+			return nil
+		}
+	}
+
+	f, err := c.Fs.Open(id)
+
+	if err != nil {
+		return nil
+	}
+
+	return f
+}
+
+func (c *Cache) isExpired(modTime time.Time) bool {
+	if c.maxAge < 0 {
+		return false
+	}
+	return c.maxAge == 0 || time.Now().Sub(modTime) > c.maxAge
+}
+
+// For testing
+func (c *Cache) getString(id string) string {
+	id = cleanID(id)
+
+	c.nlocker.Lock(id)
+	defer c.nlocker.Unlock(id)
+
+	f, err := c.Fs.Open(id)
+
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+
+	b, _ := ioutil.ReadAll(f)
+	return string(b)
+
+}
+
+// Caches is a named set of caches.
+type Caches map[string]*Cache
+
+// Get gets a named cache, nil if none found.
+func (f Caches) Get(name string) *Cache {
+	return f[strings.ToLower(name)]
+}
+
+// NewCaches creates a new set of file caches from the given
+// configuration.
+func NewCaches(p *helpers.PathSpec) (Caches, error) {
+	dcfg, err := decodeConfig(p)
+	if err != nil {
+		return nil, err
+	}
+
+	fs := p.Fs.Source
+
+	m := make(Caches)
+	for k, v := range dcfg {
+		var cfs afero.Fs
+
+		if v.isResourceDir {
+			cfs = p.BaseFs.Resources.Fs
+		} else {
+			cfs = fs
+		}
+
+		var baseDir string
+		if !strings.HasPrefix(v.Dir, "_gen") {
+			// We do cache eviction (file removes) and since the user can set
+			// his/hers own cache directory, we really want to make sure
+			// we do not delete any files that do not belong to this cache.
+			// We do add the cache name as the root, but this is an extra safe
+			// guard. We skip the files inside /resources/_gen/ because
+			// that would be breaking.
+			baseDir = filepath.Join(v.Dir, filecacheRootDirname, k)
+		} else {
+			baseDir = filepath.Join(v.Dir, k)
+		}
+		if err = cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) {
+			return nil, err
+		}
+
+		bfs := afero.NewBasePathFs(cfs, baseDir)
+
+		m[k] = NewCache(bfs, v.MaxAge)
+	}
+
+	return m, nil
+}
+
+func cleanID(name string) string {
+	return strings.TrimPrefix(filepath.Clean(name), helpers.FilePathSeparator)
+}

cache/filecache/filecache_config.go

@@ -0,0 +1,202 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/gohugoio/hugo/helpers"
+
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/spf13/afero"
+)
+
+const (
+	cachesConfigKey = "caches"
+
+	resourcesGenDir = ":resourceDir/_gen"
+)
+
+var defaultCacheConfig = cacheConfig{
+	MaxAge: -1, // Never expire
+	Dir:    ":cacheDir/:project",
+}
+
+const (
+	cacheKeyGetJSON = "getjson"
+	cacheKeyGetCSV  = "getcsv"
+	cacheKeyImages  = "images"
+	cacheKeyAssets  = "assets"
+)
+
+var defaultCacheConfigs = map[string]cacheConfig{
+	cacheKeyGetJSON: defaultCacheConfig,
+	cacheKeyGetCSV:  defaultCacheConfig,
+	cacheKeyImages: cacheConfig{
+		MaxAge: -1,
+		Dir:    resourcesGenDir,
+	},
+	cacheKeyAssets: cacheConfig{
+		MaxAge: -1,
+		Dir:    resourcesGenDir,
+	},
+}
+
+type cachesConfig map[string]cacheConfig
+
+type cacheConfig struct {
+	// Max age of cache entries in this cache. Any items older than this will
+	// be removed and not returned from the cache.
+	// a negative value means forever, 0 means cache is disabled.
+	MaxAge time.Duration
+
+	// The directory where files are stored.
+	Dir string
+
+	// Will resources/_gen will get its own composite filesystem that
+	// also checks any theme.
+	isResourceDir bool
+}
+
+// GetJSONCache gets the file cache for getJSON.
+func (f Caches) GetJSONCache() *Cache {
+	return f[cacheKeyGetJSON]
+}
+
+// GetCSVCache gets the file cache for getCSV.
+func (f Caches) GetCSVCache() *Cache {
+	return f[cacheKeyGetCSV]
+}
+
+// ImageCache gets the file cache for processed images.
+func (f Caches) ImageCache() *Cache {
+	return f[cacheKeyImages]
+}
+
+// AssetsCache gets the file cache for assets (processed resources, SCSS etc.).
+func (f Caches) AssetsCache() *Cache {
+	return f[cacheKeyAssets]
+}
+
+func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
+	c := make(cachesConfig)
+	valid := make(map[string]bool)
+	// Add defaults
+	for k, v := range defaultCacheConfigs {
+		c[k] = v
+		valid[k] = true
+	}
+
+	cfg := p.Cfg
+
+	m := cfg.GetStringMap(cachesConfigKey)
+
+	_, isOsFs := p.Fs.Source.(*afero.OsFs)
+
+	for k, v := range m {
+		cc := defaultCacheConfig
+
+		dc := &mapstructure.DecoderConfig{
+			Result:           &cc,
+			DecodeHook:       mapstructure.StringToTimeDurationHookFunc(),
+			WeaklyTypedInput: true,
+		}
+
+		decoder, err := mapstructure.NewDecoder(dc)
+		if err != nil {
+			return c, err
+		}
+
+		if err := decoder.Decode(v); err != nil {
+			return nil, err
+		}
+
+		if cc.Dir == "" {
+			return c, errors.New("must provide cache Dir")
+		}
+
+		name := strings.ToLower(k)
+		if !valid[name] {
+			return nil, errors.Errorf("%q is not a valid cache name", name)
+		}
+
+		c[name] = cc
+	}
+
+	// This is a very old flag in Hugo, but we need to respect it.
+	disabled := cfg.GetBool("ignoreCache")
+
+	for k, v := range c {
+		dir := filepath.ToSlash(filepath.Clean(v.Dir))
+		hadSlash := strings.HasPrefix(dir, "/")
+		parts := strings.Split(dir, "/")
+
+		for i, part := range parts {
+			if strings.HasPrefix(part, ":") {
+				resolved, isResource, err := resolveDirPlaceholder(p, part)
+				if err != nil {
+					return c, err
+				}
+				if isResource {
+					v.isResourceDir = true
+				}
+				parts[i] = resolved
+			}
+		}
+
+		dir = path.Join(parts...)
+		if hadSlash {
+			dir = "/" + dir
+		}
+		v.Dir = filepath.Clean(filepath.FromSlash(dir))
+
+		if !v.isResourceDir {
+			if isOsFs && !filepath.IsAbs(v.Dir) {
+				return c, errors.Errorf("%q must resolve to an absolute directory", v.Dir)
+			}
+
+			// Avoid cache in root, e.g. / (Unix) or c:\ (Windows)
+			if len(strings.TrimPrefix(v.Dir, filepath.VolumeName(v.Dir))) == 1 {
+				return c, errors.Errorf("%q is a root folder and not allowed as cache dir", v.Dir)
+			}
+		}
+
+		if disabled {
+			v.MaxAge = 0
+		}
+
+		c[k] = v
+	}
+
+	return c, nil
+}
+
+// Resolves :resourceDir => /myproject/resources etc., :cacheDir => ...
+func resolveDirPlaceholder(p *helpers.PathSpec, placeholder string) (cacheDir string, isResource bool, err error) {
+	switch strings.ToLower(placeholder) {
+	case ":resourcedir":
+		return "", true, nil
+	case ":cachedir":
+		d, err := helpers.GetCacheDir(p.Fs.Source, p.Cfg)
+		return d, false, err
+	case ":project":
+		return filepath.Base(p.WorkingDir), false, nil
+	}
+
+	return "", false, errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder)
+}

cache/filecache/filecache_config_test.go

@@ -0,0 +1,207 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/gohugoio/hugo/helpers"
+
+	"github.com/gohugoio/hugo/config"
+	"github.com/gohugoio/hugo/hugofs"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDecodeConfig(t *testing.T) {
+	t.Parallel()
+
+	assert := require.New(t)
+
+	configStr := `
+resourceDir = "myresources"
+contentDir = "content"
+dataDir = "data"
+i18nDir = "i18n"
+layoutDir = "layouts"
+assetDir = "assets"
+archetypeDir = "archetypes"
+
+[caches]
+[caches.getJSON]
+maxAge = "10m"
+dir = "/path/to/c1"
+[caches.getCSV]
+maxAge = "11h"
+dir = "/path/to/c2"
+[caches.images]
+dir = "/path/to/c3"
+
+`
+
+	cfg, err := config.FromConfigString(configStr, "toml")
+	assert.NoError(err)
+	fs := hugofs.NewMem(cfg)
+	p, err := helpers.NewPathSpec(fs, cfg)
+	assert.NoError(err)
+
+	decoded, err := decodeConfig(p)
+	assert.NoError(err)
+
+	assert.Equal(4, len(decoded))
+
+	c2 := decoded["getcsv"]
+	assert.Equal("11h0m0s", c2.MaxAge.String())
+	assert.Equal(filepath.FromSlash("/path/to/c2"), c2.Dir)
+
+	c3 := decoded["images"]
+	assert.Equal(time.Duration(-1), c3.MaxAge)
+	assert.Equal(filepath.FromSlash("/path/to/c3"), c3.Dir)
+
+}
+
+func TestDecodeConfigIgnoreCache(t *testing.T) {
+	t.Parallel()
+
+	assert := require.New(t)
+
+	configStr := `
+resourceDir = "myresources"
+contentDir = "content"
+dataDir = "data"
+i18nDir = "i18n"
+layoutDir = "layouts"
+assetDir = "assets"
+archeTypedir = "archetypes"
+
+ignoreCache = true
+[caches]
+[caches.getJSON]
+maxAge = 1234
+dir = "/path/to/c1"
+[caches.getCSV]
+maxAge = 3456
+dir = "/path/to/c2"
+[caches.images]
+dir = "/path/to/c3"
+
+`
+
+	cfg, err := config.FromConfigString(configStr, "toml")
+	assert.NoError(err)
+	fs := hugofs.NewMem(cfg)
+	p, err := helpers.NewPathSpec(fs, cfg)
+	assert.NoError(err)
+
+	decoded, err := decodeConfig(p)
+	assert.NoError(err)
+
+	assert.Equal(4, len(decoded))
+
+	for _, v := range decoded {
+		assert.Equal(time.Duration(0), v.MaxAge)
+	}
+
+}
+
+func TestDecodeConfigDefault(t *testing.T) {
+	assert := require.New(t)
+	cfg := newTestConfig()
+
+	if runtime.GOOS == "windows" {
+		cfg.Set("resourceDir", "c:\\cache\\resources")
+		cfg.Set("cacheDir", "c:\\cache\\thecache")
+
+	} else {
+		cfg.Set("resourceDir", "/cache/resources")
+		cfg.Set("cacheDir", "/cache/thecache")
+	}
+
+	fs := hugofs.NewMem(cfg)
+	p, err := helpers.NewPathSpec(fs, cfg)
+	assert.NoError(err)
+
+	decoded, err := decodeConfig(p)
+
+	assert.NoError(err)
+
+	assert.Equal(4, len(decoded))
+
+	imgConfig := decoded[cacheKeyImages]
+	jsonConfig := decoded[cacheKeyGetJSON]
+
+	if runtime.GOOS == "windows" {
+		assert.Equal("_gen", imgConfig.Dir)
+	} else {
+		assert.Equal("_gen", imgConfig.Dir)
+		assert.Equal("/cache/thecache/hugoproject", jsonConfig.Dir)
+	}
+
+	assert.True(imgConfig.isResourceDir)
+	assert.False(jsonConfig.isResourceDir)
+}
+
+func TestDecodeConfigInvalidDir(t *testing.T) {
+	t.Parallel()
+
+	assert := require.New(t)
+
+	configStr := `
+resourceDir = "myresources"
+contentDir = "content"
+dataDir = "data"
+i18nDir = "i18n"
+layoutDir = "layouts"
+assetDir = "assets"
+archeTypedir = "archetypes"
+
+[caches]
+[caches.getJSON]
+maxAge = "10m"
+dir = "/"
+
+`
+	if runtime.GOOS == "windows" {
+		configStr = strings.Replace(configStr, "/", "c:\\\\", 1)
+	}
+
+	cfg, err := config.FromConfigString(configStr, "toml")
+	assert.NoError(err)
+	fs := hugofs.NewMem(cfg)
+	p, err := helpers.NewPathSpec(fs, cfg)
+	assert.NoError(err)
+
+	_, err = decodeConfig(p)
+	assert.Error(err)
+
+}
+
+func newTestConfig() *viper.Viper {
+	cfg := viper.New()
+	cfg.Set("workingDir", filepath.FromSlash("/my/cool/hugoproject"))
+	cfg.Set("contentDir", "content")
+	cfg.Set("dataDir", "data")
+	cfg.Set("resourceDir", "resources")
+	cfg.Set("i18nDir", "i18n")
+	cfg.Set("layoutDir", "layouts")
+	cfg.Set("archetypeDir", "archetypes")
+	cfg.Set("assetDir", "assets")
+
+	return cfg
+}

cache/filecache/filecache_pruner.go

@@ -0,0 +1,80 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"io"
+	"os"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/afero"
+)
+
+// Prune removes expired and unused items from this cache.
+// The last one requires a full build so the cache usage can be tracked.
+// Note that we operate directly on the filesystem here, so this is not
+// thread safe.
+func (c Caches) Prune() (int, error) {
+	counter := 0
+	for k, cache := range c {
+		err := afero.Walk(cache.Fs, "", func(name string, info os.FileInfo, err error) error {
+			if info == nil {
+				return nil
+			}
+
+			name = cleanID(name)
+
+			if info.IsDir() {
+				f, err := cache.Fs.Open(name)
+				if err != nil {
+					// This cache dir may not exist.
+					return nil
+				}
+				defer f.Close()
+				_, err = f.Readdirnames(1)
+				if err == io.EOF {
+					// Empty dir.
+					return cache.Fs.Remove(name)
+				}
+
+				return nil
+			}
+
+			shouldRemove := cache.isExpired(info.ModTime())
+
+			if !shouldRemove && len(cache.nlocker.seen) > 0 {
+				// Remove it if it's not been touched/used in the last build.
+				_, seen := cache.nlocker.seen[name]
+				shouldRemove = !seen
+			}
+
+			if shouldRemove {
+				err := cache.Fs.Remove(name)
+				if err == nil {
+					counter++
+				}
+				return err
+			}
+
+			return nil
+		})
+
+		if err != nil {
+			return counter, errors.Wrapf(err, "failed to prune cache %q", k)
+		}
+
+	}
+
+	return counter, nil
+}

cache/filecache/filecache_pruner_test.go

@@ -0,0 +1,118 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/gohugoio/hugo/config"
+	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/hugofs"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestPrune(t *testing.T) {
+	t.Parallel()
+
+	assert := require.New(t)
+
+	configStr := `
+resourceDir = "myresources"
+contentDir = "content"
+dataDir = "data"
+i18nDir = "i18n"
+layoutDir = "layouts"
+assetDir = "assets"
+archeTypedir = "archetypes"
+
+[caches]
+[caches.getjson]
+maxAge = "200ms"
+dir = "/cache/c"
+[caches.getcsv]
+maxAge = "200ms"
+dir = "/cache/d"
+[caches.assets]
+maxAge = "200ms"
+dir = ":resourceDir/_gen"
+[caches.images]
+maxAge = "200ms"
+dir = ":resourceDir/_gen"
+`
+
+	cfg, err := config.FromConfigString(configStr, "toml")
+	assert.NoError(err)
+
+	for _, name := range []string{cacheKeyGetCSV, cacheKeyGetJSON, cacheKeyAssets, cacheKeyImages} {
+		msg := fmt.Sprintf("cache: %s", name)
+		fs := hugofs.NewMem(cfg)
+		p, err := helpers.NewPathSpec(fs, cfg)
+		assert.NoError(err)
+		caches, err := NewCaches(p)
+		assert.NoError(err)
+		cache := caches[name]
+		for i := 0; i < 10; i++ {
+			id := fmt.Sprintf("i%d", i)
+			cache.GetOrCreateBytes(id, func() ([]byte, error) {
+				return []byte("abc"), nil
+			})
+			if i == 4 {
+				// This will expire the first 5
+				time.Sleep(201 * time.Millisecond)
+			}
+		}
+
+		count, err := caches.Prune()
+		assert.NoError(err)
+		assert.Equal(5, count, msg)
+
+		for i := 0; i < 10; i++ {
+			id := fmt.Sprintf("i%d", i)
+			v := cache.getString(id)
+			if i < 5 {
+				assert.Equal("", v, id)
+			} else {
+				assert.Equal("abc", v, id)
+			}
+		}
+
+		caches, err = NewCaches(p)
+		assert.NoError(err)
+		cache = caches[name]
+		// Touch one and then prune.
+		cache.GetOrCreateBytes("i5", func() ([]byte, error) {
+			return []byte("abc"), nil
+		})
+
+		count, err = caches.Prune()
+		assert.NoError(err)
+		assert.Equal(4, count)
+
+		// Now only the i5 should be left.
+		for i := 0; i < 10; i++ {
+			id := fmt.Sprintf("i%d", i)
+			v := cache.getString(id)
+			if i != 5 {
+				assert.Equal("", v, id)
+			} else {
+				assert.Equal("abc", v, id)
+			}
+		}
+
+	}
+
+}

cache/filecache/filecache_test.go

@@ -0,0 +1,231 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/gohugoio/hugo/common/hugio"
+	"github.com/gohugoio/hugo/config"
+	"github.com/gohugoio/hugo/helpers"
+
+	"github.com/gohugoio/hugo/hugofs"
+	"github.com/spf13/afero"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFileCache(t *testing.T) {
+	t.Parallel()
+	assert := require.New(t)
+
+	for _, cacheDir := range []string{"mycache", ""} {
+
+		configStr := `
+workingDir = "/my/work"
+resourceDir = "resources"
+cacheDir = "CACHEDIR"
+contentDir = "content"
+dataDir = "data"
+i18nDir = "i18n"
+layoutDir = "layouts"
+assetDir = "assets"
+archeTypedir = "archetypes"
+
+[caches]
+[caches.getJSON]
+maxAge = "10h"
+dir = ":cacheDir/c"
+
+`
+		configStr = strings.Replace(configStr, "CACHEDIR", cacheDir, 1)
+
+		cfg, err := config.FromConfigString(configStr, "toml")
+		assert.NoError(err)
+
+		fs := hugofs.NewMem(cfg)
+		p, err := helpers.NewPathSpec(fs, cfg)
+		assert.NoError(err)
+
+		caches, err := NewCaches(p)
+		assert.NoError(err)
+
+		c := caches.Get("GetJSON")
+		assert.NotNil(c)
+		assert.Equal("10h0m0s", c.maxAge.String())
+
+		bfs, ok := c.Fs.(*afero.BasePathFs)
+		assert.True(ok)
+		filename, err := bfs.RealPath("key")
+		assert.NoError(err)
+		if cacheDir != "" {
+			assert.Equal(filepath.FromSlash(cacheDir+"/c/"+filecacheRootDirname+"/getjson/key"), filename)
+		} else {
+			// Temp dir.
+			assert.Regexp(regexp.MustCompile(".*hugo_cache.*"+filecacheRootDirname+".*key"), filename)
+		}
+
+		c = caches.Get("images")
+		assert.NotNil(c)
+		assert.Equal(time.Duration(-1), c.maxAge)
+		bfs, ok = c.Fs.(*afero.BasePathFs)
+		assert.True(ok)
+		filename, _ = bfs.RealPath("key")
+		assert.Equal(filepath.FromSlash("_gen/images/key"), filename)
+
+		rf := func(s string) func() (io.ReadCloser, error) {
+			return func() (io.ReadCloser, error) {
+				return struct {
+					io.ReadSeeker
+					io.Closer
+				}{
+					strings.NewReader(s),
+					ioutil.NopCloser(nil),
+				}, nil
+			}
+		}
+
+		bf := func() ([]byte, error) {
+			return []byte("bcd"), nil
+		}
+
+		for i := 0; i < 2; i++ {
+			info, r, err := c.GetOrCreate("a", rf("abc"))
+			assert.NoError(err)
+			assert.NotNil(r)
+			assert.Equal("a", info.Name)
+			b, _ := ioutil.ReadAll(r)
+			r.Close()
+			assert.Equal("abc", string(b))
+
+			info, b, err = c.GetOrCreateBytes("b", bf)
+			assert.NoError(err)
+			assert.NotNil(r)
+			assert.Equal("b", info.Name)
+			assert.Equal("bcd", string(b))
+
+			_, b, err = c.GetOrCreateBytes("a", bf)
+			assert.NoError(err)
+			assert.Equal("abc", string(b))
+
+			_, r, err = c.GetOrCreate("a", rf("bcd"))
+			assert.NoError(err)
+			b, _ = ioutil.ReadAll(r)
+			r.Close()
+			assert.Equal("abc", string(b))
+		}
+
+		assert.NotNil(caches.Get("getJSON"))
+
+		info, w, err := caches.ImageCache().WriteCloser("mykey")
+		assert.NoError(err)
+		assert.Equal("mykey", info.Name)
+		io.WriteString(w, "Hugo is great!")
+		w.Close()
+		assert.Equal("Hugo is great!", caches.ImageCache().getString("mykey"))
+
+		info, r, err := caches.ImageCache().Get("mykey")
+		assert.NoError(err)
+		assert.NotNil(r)
+		assert.Equal("mykey", info.Name)
+		b, _ := ioutil.ReadAll(r)
+		r.Close()
+		assert.Equal("Hugo is great!", string(b))
+
+		info, b, err = caches.ImageCache().GetBytes("mykey")
+		assert.NoError(err)
+		assert.Equal("mykey", info.Name)
+		assert.Equal("Hugo is great!", string(b))
+
+	}
+
+}
+
+func TestFileCacheConcurrent(t *testing.T) {
+	t.Parallel()
+
+	assert := require.New(t)
+
+	configStr := `
+resourceDir = "myresources"
+contentDir = "content"
+dataDir = "data"
+i18nDir = "i18n"
+layoutDir = "layouts"
+assetDir = "assets"
+archeTypedir = "archetypes"
+
+[caches]
+[caches.getjson]
+maxAge = "1s"
+dir = "/cache/c"
+
+`
+
+	cfg, err := config.FromConfigString(configStr, "toml")
+	assert.NoError(err)
+	fs := hugofs.NewMem(cfg)
+	p, err := helpers.NewPathSpec(fs, cfg)
+	assert.NoError(err)
+
+	caches, err := NewCaches(p)
+	assert.NoError(err)
+
+	const cacheName = "getjson"
+
+	filenameData := func(i int) (string, string) {
+		data := fmt.Sprintf("data: %d", i)
+		filename := fmt.Sprintf("file%d", i)
+		return filename, data
+	}
+
+	var wg sync.WaitGroup
+
+	for i := 0; i < 50; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			for j := 0; j < 20; j++ {
+				c := caches.Get(cacheName)
+				assert.NotNil(c)
+				filename, data := filenameData(i)
+				_, r, err := c.GetOrCreate(filename, func() (io.ReadCloser, error) {
+					return hugio.ToReadCloser(strings.NewReader(data)), nil
+				})
+				assert.NoError(err)
+				b, _ := ioutil.ReadAll(r)
+				r.Close()
+				assert.Equal(data, string(b))
+				// Trigger some expiration.
+				time.Sleep(50 * time.Millisecond)
+			}
+		}(i)
+
+	}
+	wg.Wait()
+}
+
+func TestCleanID(t *testing.T) {
+	assert := require.New(t)
+	assert.Equal(filepath.FromSlash("a/b/c.txt"), cleanID(filepath.FromSlash("/a/b//c.txt")))
+	assert.Equal(filepath.FromSlash("a/b/c.txt"), cleanID(filepath.FromSlash("a/b//c.txt")))
+}

cache/partitioned_lazy_cache.go

@@ -24,34 +24,52 @@ type Partition struct {
 	Load func() (map[string]interface{}, error)
 }
 
-type lazyPartition struct {
+// Lazy represents a lazily loaded cache.
+type Lazy struct {
 	initSync sync.Once
+	initErr  error
 	cache    map[string]interface{}
 	load     func() (map[string]interface{}, error)
 }
 
-func (l *lazyPartition) init() error {
-	var err error
+// NewLazy creates a lazy cache with the given load func.
+func NewLazy(load func() (map[string]interface{}, error)) *Lazy {
+	return &Lazy{load: load}
+}
+
+func (l *Lazy) init() error {
 	l.initSync.Do(func() {
-		var c map[string]interface{}
-		c, err = l.load()
+		c, err := l.load()
 		l.cache = c
+		l.initErr = err
+
 	})
 
-	return err
+	return l.initErr
+}
+
+// Get initializes the cache if not already initialized, then looks up the
+// given key.
+func (l *Lazy) Get(key string) (interface{}, bool, error) {
+	l.init()
+	if l.initErr != nil {
+		return nil, false, l.initErr
+	}
+	v, found := l.cache[key]
+	return v, found, nil
 }
 
 // PartitionedLazyCache is a lazily loaded cache paritioned by a supplied string key.
 type PartitionedLazyCache struct {
-	partitions map[string]*lazyPartition
+	partitions map[string]*Lazy
 }
 
 // NewPartitionedLazyCache creates a new NewPartitionedLazyCache with the supplied
 // partitions.
 func NewPartitionedLazyCache(partitions ...Partition) *PartitionedLazyCache {
-	lazyPartitions := make(map[string]*lazyPartition, len(partitions))
+	lazyPartitions := make(map[string]*Lazy, len(partitions))
 	for _, partition := range partitions {
-		lazyPartitions[partition.Key] = &lazyPartition{load: partition.Load}
+		lazyPartitions[partition.Key] = NewLazy(partition.Load)
 	}
 	cache := &PartitionedLazyCache{partitions: lazyPartitions}
 
@@ -67,11 +85,12 @@ func (c *PartitionedLazyCache) Get(partition, key string) (interface{}, error) {
 		return nil, nil
 	}
 
-	if err := p.init(); err != nil {
+	v, found, err := p.Get(key)
+	if err != nil {
 		return nil, err
 	}
 
-	if v, found := p.cache[key]; found {
+	if found {
 		return v, nil
 	}
 

commands/benchmark.go

@@ -23,51 +23,55 @@ import (
 	jww "github.com/spf13/jwalterweatherman"
 )
 
-var (
+type benchmarkCmd struct {
 	benchmarkTimes int
 	cpuProfileFile string
 	memProfileFile string
-)
 
-var benchmarkCmd = &cobra.Command{
-	Use:   "benchmark",
-	Short: "Benchmark Hugo by building a site a number of times.",
-	Long: `Hugo can build a site many times over and analyze the running process
+	*baseBuilderCmd
+}
+
+func (b *commandsBuilder) newBenchmarkCmd() *benchmarkCmd {
+	cmd := &cobra.Command{
+		Use:   "benchmark",
+		Short: "Benchmark Hugo by building a site a number of times.",
+		Long: `Hugo can build a site many times over and analyze the running process
 creating a benchmark.`,
+	}
+
+	c := &benchmarkCmd{baseBuilderCmd: b.newBuilderCmd(cmd)}
+
+	cmd.Flags().StringVar(&c.cpuProfileFile, "cpuprofile", "", "path/filename for the CPU profile file")
+	cmd.Flags().StringVar(&c.memProfileFile, "memprofile", "", "path/filename for the memory profile file")
+	cmd.Flags().IntVarP(&c.benchmarkTimes, "count", "n", 13, "number of times to build the site")
+	cmd.Flags().Bool("renderToMemory", false, "render to memory (only useful for benchmark testing)")
+
+	cmd.RunE = c.benchmark
+
+	return c
 }
 
-func init() {
-	initHugoBuilderFlags(benchmarkCmd)
-	initBenchmarkBuildingFlags(benchmarkCmd)
-
-	benchmarkCmd.Flags().StringVar(&cpuProfileFile, "cpuprofile", "", "path/filename for the CPU profile file")
-	benchmarkCmd.Flags().StringVar(&memProfileFile, "memprofile", "", "path/filename for the memory profile file")
-	benchmarkCmd.Flags().IntVarP(&benchmarkTimes, "count", "n", 13, "number of times to build the site")
-
-	benchmarkCmd.RunE = benchmark
-}
-
-func benchmark(cmd *cobra.Command, args []string) error {
+func (c *benchmarkCmd) benchmark(cmd *cobra.Command, args []string) error {
 	cfgInit := func(c *commandeer) error {
-		c.Set("renderToMemory", renderToMemory)
 		return nil
 	}
-	c, err := InitializeConfig(false, cfgInit, benchmarkCmd)
+
+	comm, err := initializeConfig(true, false, &c.hugoBuilderCommon, c, cfgInit)
 	if err != nil {
 		return err
 	}
 
 	var memProf *os.File
-	if memProfileFile != "" {
-		memProf, err = os.Create(memProfileFile)
+	if c.memProfileFile != "" {
+		memProf, err = os.Create(c.memProfileFile)
 		if err != nil {
 			return err
 		}
 	}
 
 	var cpuProf *os.File
-	if cpuProfileFile != "" {
-		cpuProf, err = os.Create(cpuProfileFile)
+	if c.cpuProfileFile != "" {
+		cpuProf, err = os.Create(c.cpuProfileFile)
 		if err != nil {
 			return err
 		}
@@ -82,8 +86,8 @@ func benchmark(cmd *cobra.Command, args []string) error {
 	}
 
 	t := time.Now()
-	for i := 0; i < benchmarkTimes; i++ {
-		if err = c.resetAndBuildSites(); err != nil {
+	for i := 0; i < c.benchmarkTimes; i++ {
+		if err = comm.resetAndBuildSites(); err != nil {
 			return err
 		}
 	}
@@ -103,9 +107,9 @@ func benchmark(cmd *cobra.Command, args []string) error {
 	totalMallocs := memStats.Mallocs - mallocs
 
 	jww.FEEDBACK.Println()
-	jww.FEEDBACK.Printf("Average time per operation: %vms\n", int(1000*totalTime.Seconds()/float64(benchmarkTimes)))
-	jww.FEEDBACK.Printf("Average memory allocated per operation: %vkB\n", totalMemAllocated/uint64(benchmarkTimes)/1024)
-	jww.FEEDBACK.Printf("Average allocations per operation: %v\n", totalMallocs/uint64(benchmarkTimes))
+	jww.FEEDBACK.Printf("Average time per operation: %vms\n", int(1000*totalTime.Seconds()/float64(c.benchmarkTimes)))
+	jww.FEEDBACK.Printf("Average memory allocated per operation: %vkB\n", totalMemAllocated/uint64(c.benchmarkTimes)/1024)
+	jww.FEEDBACK.Printf("Average allocations per operation: %v\n", totalMallocs/uint64(c.benchmarkTimes))
 
 	return nil
 }

commands/check.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,13 +11,24 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// +build !darwin
+
 package commands
 
 import (
 	"github.com/spf13/cobra"
 )
 
-var checkCmd = &cobra.Command{
-	Use:   "check",
-	Short: "Contains some verification checks",
+var _ cmder = (*checkCmd)(nil)
+
+type checkCmd struct {
+	*baseCmd
+}
+
+func newCheckCmd() *checkCmd {
+	return &checkCmd{baseCmd: &baseCmd{cmd: &cobra.Command{
+		Use:   "check",
+		Short: "Contains some verification checks",
+	},
+	}}
 }

commands/check_darwin.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,18 +11,26 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package transform
+package commands
 
-var ar = newAbsURLReplacer()
+import (
+	"github.com/spf13/cobra"
+)
 
-// AbsURL replaces relative URLs with absolute ones
-// in HTML files, using the baseURL setting.
-var AbsURL = func(ct contentTransformer) {
-	ar.replaceInHTML(ct)
+var _ cmder = (*checkCmd)(nil)
+
+type checkCmd struct {
+	*baseCmd
 }
 
-// AbsURLInXML replaces relative URLs with absolute ones
-// in XML files, using the baseURL setting.
-var AbsURLInXML = func(ct contentTransformer) {
-	ar.replaceInXML(ct)
+func newCheckCmd() *checkCmd {
+	cc := &checkCmd{baseCmd: &baseCmd{cmd: &cobra.Command{
+		Use:   "check",
+		Short: "Contains some verification checks",
+	},
+	}}
+
+	cc.cmd.AddCommand(newLimitCmd().getCommand())
+
+	return cc
 }

commands/commandeer.go

@@ -1,4 +1,4 @@
-// Copyright 2017 The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,24 +14,111 @@
 package commands
 
 import (
+	"bytes"
+	"errors"
+
+	"github.com/gohugoio/hugo/common/herrors"
+
+	"io/ioutil"
+
+	jww "github.com/spf13/jwalterweatherman"
+
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gohugoio/hugo/common/loggers"
+	"github.com/gohugoio/hugo/config"
+
+	"github.com/spf13/cobra"
+
+	"github.com/gohugoio/hugo/hugolib"
+	"github.com/spf13/afero"
+
+	"github.com/bep/debounce"
 	"github.com/gohugoio/hugo/common/types"
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
-	src "github.com/gohugoio/hugo/source"
+	"github.com/gohugoio/hugo/langs"
 )
 
-type commandeer struct {
+type commandeerHugoState struct {
 	*deps.DepsCfg
-	pathSpec    *helpers.PathSpec
+	hugo     *hugolib.HugoSites
+	fsCreate sync.Once
+}
+
+type commandeer struct {
+	*commandeerHugoState
+
+	logger *loggers.Logger
+
+	// Currently only set when in "fast render mode". But it seems to
+	// be fast enough that we could maybe just add it for all server modes.
+	changeDetector *fileChangeDetector
+
+	// We need to reuse this on server rebuilds.
+	destinationFs afero.Fs
+
+	h    *hugoBuilderCommon
+	ftch flagsToConfigHandler
+
 	visitedURLs *types.EvictingStringQueue
 
-	staticDirsConfig []*src.Dirs
+	doWithCommandeer func(c *commandeer) error
 
-	serverPorts []int
-	languages   helpers.Languages
+	// We watch these for changes.
+	configFiles []string
+
+	// Used in cases where we get flooded with events in server mode.
+	debounce func(f func())
+
+	serverPorts         []int
+	languagesConfigured bool
+	languages           langs.Languages
+	doLiveReload        bool
+	fastRenderMode      bool
+	showErrorInBrowser  bool
 
 	configured bool
+	paused     bool
+
+	// Any error from the last build.
+	buildErr error
+}
+
+func (c *commandeer) errCount() int {
+	return int(c.logger.ErrorCounter.Count())
+}
+
+func (c *commandeer) getErrorWithContext() interface{} {
+	errCount := c.errCount()
+
+	if errCount == 0 {
+		return nil
+	}
+
+	m := make(map[string]interface{})
+
+	m["Error"] = errors.New(removeErrorPrefixFromLog(c.logger.Errors()))
+	m["Version"] = hugoVersionString()
+
+	fe := herrors.UnwrapErrorWithFileContext(c.buildErr)
+	if fe != nil {
+		m["File"] = fe
+	}
+
+	if c.h.verbose {
+		var b bytes.Buffer
+		herrors.FprintStackTrace(&b, c.buildErr)
+		m["StackTrace"] = b.String()
+	}
+
+	return m
 }
 
 func (c *commandeer) Set(key string, value interface{}) {
@@ -41,40 +128,260 @@ func (c *commandeer) Set(key string, value interface{}) {
 	c.Cfg.Set(key, value)
 }
 
-// PathSpec lazily creates a new PathSpec, as all the paths must
-// be configured before it is created.
-func (c *commandeer) PathSpec() *helpers.PathSpec {
-	c.configured = true
-	return c.pathSpec
-}
-
 func (c *commandeer) initFs(fs *hugofs.Fs) error {
+	c.destinationFs = fs.Destination
 	c.DepsCfg.Fs = fs
-	ps, err := helpers.NewPathSpec(fs, c.Cfg)
-	if err != nil {
-		return err
-	}
-	c.pathSpec = ps
-
-	dirsConfig, err := c.createStaticDirsConfig()
-	if err != nil {
-		return err
-	}
-	c.staticDirsConfig = dirsConfig
 
 	return nil
 }
 
-func newCommandeer(cfg *deps.DepsCfg, running bool) (*commandeer, error) {
-	cfg.Running = running
+func newCommandeer(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f flagsToConfigHandler, doWithCommandeer func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) {
 
-	var languages helpers.Languages
-
-	if l, ok := cfg.Cfg.Get("languagesSorted").(helpers.Languages); ok {
-		languages = l
+	var rebuildDebouncer func(f func())
+	if running {
+		// The time value used is tested with mass content replacements in a fairly big Hugo site.
+		// It is better to wait for some seconds in those cases rather than get flooded
+		// with rebuilds.
+		rebuildDebouncer, _, _ = debounce.New(4 * time.Second)
 	}
 
-	c := &commandeer{DepsCfg: cfg, languages: languages, visitedURLs: types.NewEvictingStringQueue(10)}
+	c := &commandeer{
+		h:                   h,
+		ftch:                f,
+		commandeerHugoState: &commandeerHugoState{},
+		doWithCommandeer:    doWithCommandeer,
+		visitedURLs:         types.NewEvictingStringQueue(10),
+		debounce:            rebuildDebouncer,
+		// This will be replaced later, but we need something to log to before the configuration is read.
+		logger: loggers.NewLogger(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, running),
+	}
+
+	return c, c.loadConfig(mustHaveConfigFile, running)
+}
+
+type fileChangeDetector struct {
+	sync.Mutex
+	current map[string]string
+	prev    map[string]string
+
+	irrelevantRe *regexp.Regexp
+}
+
+func (f *fileChangeDetector) OnFileClose(name, md5sum string) {
+	f.Lock()
+	defer f.Unlock()
+	f.current[name] = md5sum
+}
+
+func (f *fileChangeDetector) changed() []string {
+	if f == nil {
+		return nil
+	}
+	f.Lock()
+	defer f.Unlock()
+	var c []string
+	for k, v := range f.current {
+		vv, found := f.prev[k]
+		if !found || v != vv {
+			c = append(c, k)
+		}
+	}
+
+	return f.filterIrrelevant(c)
+}
+
+func (f *fileChangeDetector) filterIrrelevant(in []string) []string {
+	var filtered []string
+	for _, v := range in {
+		if !f.irrelevantRe.MatchString(v) {
+			filtered = append(filtered, v)
+		}
+	}
+	return filtered
+}
+
+func (f *fileChangeDetector) PrepareNew() {
+	if f == nil {
+		return
+	}
+
+	f.Lock()
+	defer f.Unlock()
+
+	if f.current == nil {
+		f.current = make(map[string]string)
+		f.prev = make(map[string]string)
+		return
+	}
+
+	f.prev = make(map[string]string)
+	for k, v := range f.current {
+		f.prev[k] = v
+	}
+	f.current = make(map[string]string)
+}
+
+func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {
+
+	if c.DepsCfg == nil {
+		c.DepsCfg = &deps.DepsCfg{}
+	}
+
+	if c.logger != nil {
+		// Truncate the error log if this is a reload.
+		c.logger.Reset()
+	}
+
+	cfg := c.DepsCfg
+	c.configured = false
+	cfg.Running = running
+
+	var dir string
+	if c.h.source != "" {
+		dir, _ = filepath.Abs(c.h.source)
+	} else {
+		dir, _ = os.Getwd()
+	}
+
+	var sourceFs afero.Fs = hugofs.Os
+	if c.DepsCfg.Fs != nil {
+		sourceFs = c.DepsCfg.Fs.Source
+	}
+
+	doWithConfig := func(cfg config.Provider) error {
+
+		if c.ftch != nil {
+			c.ftch.flagsToConfig(cfg)
+		}
+
+		cfg.Set("workingDir", dir)
+
+		return nil
+	}
+
+	doWithCommandeer := func(cfg config.Provider) error {
+		c.Cfg = cfg
+		if c.doWithCommandeer == nil {
+			return nil
+		}
+		err := c.doWithCommandeer(c)
+		return err
+	}
+
+	config, configFiles, err := hugolib.LoadConfig(
+		hugolib.ConfigSourceDescriptor{Fs: sourceFs, Path: c.h.source, WorkingDir: dir, Filename: c.h.cfgFile},
+		doWithCommandeer,
+		doWithConfig)
+
+	if err != nil {
+		if mustHaveConfigFile {
+			return err
+		}
+		if err != hugolib.ErrNoConfigFile {
+			return err
+		}
+
+	}
+
+	c.configFiles = configFiles
+
+	if l, ok := c.Cfg.Get("languagesSorted").(langs.Languages); ok {
+		c.languagesConfigured = true
+		c.languages = l
+	}
+
+	// Set some commonly used flags
+	c.doLiveReload = !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload")
+	c.fastRenderMode = c.doLiveReload && !c.Cfg.GetBool("disableFastRender")
+	c.showErrorInBrowser = c.doLiveReload && !c.Cfg.GetBool("disableBrowserError")
+
+	// This is potentially double work, but we need to do this one more time now
+	// that all the languages have been configured.
+	if c.doWithCommandeer != nil {
+		if err := c.doWithCommandeer(c); err != nil {
+			return err
+		}
+	}
+
+	logger, err := c.createLogger(config, running)
+	if err != nil {
+		return err
+	}
+
+	cfg.Logger = logger
+	c.logger = logger
+
+	createMemFs := config.GetBool("renderToMemory")
+
+	if createMemFs {
+		// Rendering to memoryFS, publish to Root regardless of publishDir.
+		config.Set("publishDir", "/")
+	}
+
+	c.fsCreate.Do(func() {
+		fs := hugofs.NewFrom(sourceFs, config)
+
+		if c.destinationFs != nil {
+			// Need to reuse the destination on server rebuilds.
+			fs.Destination = c.destinationFs
+		} else if createMemFs {
+			// Hugo writes the output to memory instead of the disk.
+			fs.Destination = new(afero.MemMapFs)
+		}
+
+		if c.fastRenderMode {
+			// For now, fast render mode only. It should, however, be fast enough
+			// for the full variant, too.
+			changeDetector := &fileChangeDetector{
+				// We use this detector to decide to do a Hot reload of a single path or not.
+				// We need to filter out source maps and possibly some other to be able
+				// to make that decision.
+				irrelevantRe: regexp.MustCompile(`\.map$`),
+			}
+			changeDetector.PrepareNew()
+			fs.Destination = hugofs.NewHashingFs(fs.Destination, changeDetector)
+			c.changeDetector = changeDetector
+		}
+
+		err = c.initFs(fs)
+		if err != nil {
+			return
+		}
+
+		var h *hugolib.HugoSites
+
+		h, err = hugolib.NewHugoSites(*c.DepsCfg)
+		c.hugo = h
+
+	})
+
+	if err != nil {
+		return err
+	}
+
+	cacheDir, err := helpers.GetCacheDir(sourceFs, config)
+	if err != nil {
+		return err
+	}
+	config.Set("cacheDir", cacheDir)
+
+	cfg.Logger.INFO.Println("Using config file:", config.ConfigFileUsed())
+
+	themeDir := c.hugo.PathSpec.GetFirstThemeDir()
+	if themeDir != "" {
+		if _, err := sourceFs.Stat(themeDir); os.IsNotExist(err) {
+			return newSystemError("Unable to find theme Directory:", themeDir)
+		}
+	}
+
+	dir, themeVersionMismatch, minVersion := c.isThemeVsHugoVersionMismatch(sourceFs)
+
+	if themeVersionMismatch {
+		name := filepath.Base(dir)
+		cfg.Logger.ERROR.Printf("%s theme does not support Hugo version %s. Minimum version required is %s\n",
+			strings.ToUpper(name), helpers.CurrentHugoVersion.ReleaseVersion(), minVersion)
+	}
+
+	return nil
 
-	return c, nil
 }

commands/commands.go

@@ -0,0 +1,255 @@
+// Copyright 2017 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package commands
+
+import (
+	"github.com/gohugoio/hugo/common/loggers"
+	"github.com/gohugoio/hugo/config"
+	"github.com/gohugoio/hugo/helpers"
+	"github.com/spf13/cobra"
+	"github.com/spf13/nitro"
+)
+
+type commandsBuilder struct {
+	hugoBuilderCommon
+
+	commands []cmder
+}
+
+func newCommandsBuilder() *commandsBuilder {
+	return &commandsBuilder{}
+}
+
+func (b *commandsBuilder) addCommands(commands ...cmder) *commandsBuilder {
+	b.commands = append(b.commands, commands...)
+	return b
+}
+
+func (b *commandsBuilder) addAll() *commandsBuilder {
+	b.addCommands(
+		b.newServerCmd(),
+		newVersionCmd(),
+		newEnvCmd(),
+		newConfigCmd(),
+		newCheckCmd(),
+		b.newBenchmarkCmd(),
+		newConvertCmd(),
+		b.newNewCmd(),
+		newListCmd(),
+		newImportCmd(),
+		newGenCmd(),
+		createReleaser(),
+	)
+
+	return b
+}
+
+func (b *commandsBuilder) build() *hugoCmd {
+	h := b.newHugoCmd()
+	addCommands(h.getCommand(), b.commands...)
+	return h
+}
+
+func addCommands(root *cobra.Command, commands ...cmder) {
+	for _, command := range commands {
+		cmd := command.getCommand()
+		if cmd == nil {
+			continue
+		}
+		root.AddCommand(cmd)
+	}
+}
+
+type baseCmd struct {
+	cmd *cobra.Command
+}
+
+var _ commandsBuilderGetter = (*baseBuilderCmd)(nil)
+
+// Used in tests.
+type commandsBuilderGetter interface {
+	getCommandsBuilder() *commandsBuilder
+}
+type baseBuilderCmd struct {
+	*baseCmd
+	*commandsBuilder
+}
+
+func (b *baseBuilderCmd) getCommandsBuilder() *commandsBuilder {
+	return b.commandsBuilder
+}
+
+func (c *baseCmd) getCommand() *cobra.Command {
+	return c.cmd
+}
+
+func newBaseCmd(cmd *cobra.Command) *baseCmd {
+	return &baseCmd{cmd: cmd}
+}
+
+func (b *commandsBuilder) newBuilderCmd(cmd *cobra.Command) *baseBuilderCmd {
+	bcmd := &baseBuilderCmd{commandsBuilder: b, baseCmd: &baseCmd{cmd: cmd}}
+	bcmd.hugoBuilderCommon.handleFlags(cmd)
+	return bcmd
+}
+
+func (c *baseCmd) flagsToConfig(cfg config.Provider) {
+	initializeFlags(c.cmd, cfg)
+}
+
+type hugoCmd struct {
+	*baseBuilderCmd
+
+	// Need to get the sites once built.
+	c *commandeer
+}
+
+var _ cmder = (*nilCommand)(nil)
+
+type nilCommand struct {
+}
+
+func (c *nilCommand) getCommand() *cobra.Command {
+	return nil
+}
+
+func (c *nilCommand) flagsToConfig(cfg config.Provider) {
+
+}
+
+func (b *commandsBuilder) newHugoCmd() *hugoCmd {
+	cc := &hugoCmd{}
+
+	cc.baseBuilderCmd = b.newBuilderCmd(&cobra.Command{
+		Use:   "hugo",
+		Short: "hugo builds your site",
+		Long: `hugo is the main command, used to build your Hugo site.
+
+Hugo is a Fast and Flexible Static Site Generator
+built with love by spf13 and friends in Go.
+
+Complete documentation is available at http://gohugo.io/.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfgInit := func(c *commandeer) error {
+				if cc.buildWatch {
+					c.Set("disableLiveReload", true)
+				}
+				return nil
+			}
+
+			c, err := initializeConfig(true, cc.buildWatch, &cc.hugoBuilderCommon, cc, cfgInit)
+			if err != nil {
+				return err
+			}
+			cc.c = c
+
+			return c.build()
+		},
+	})
+
+	cc.cmd.PersistentFlags().StringVar(&cc.cfgFile, "config", "", "config file (default is path/config.yaml|json|toml)")
+	cc.cmd.PersistentFlags().BoolVar(&cc.quiet, "quiet", false, "build in quiet mode")
+
+	// Set bash-completion
+	validConfigFilenames := []string{"json", "js", "yaml", "yml", "toml", "tml"}
+	_ = cc.cmd.PersistentFlags().SetAnnotation("config", cobra.BashCompFilenameExt, validConfigFilenames)
+
+	cc.cmd.PersistentFlags().BoolVarP(&cc.verbose, "verbose", "v", false, "verbose output")
+	cc.cmd.PersistentFlags().BoolVarP(&cc.debug, "debug", "", false, "debug output")
+	cc.cmd.PersistentFlags().BoolVar(&cc.logging, "log", false, "enable Logging")
+	cc.cmd.PersistentFlags().StringVar(&cc.logFile, "logFile", "", "log File path (if set, logging enabled automatically)")
+	cc.cmd.PersistentFlags().BoolVar(&cc.verboseLog, "verboseLog", false, "verbose logging")
+
+	cc.cmd.Flags().BoolVarP(&cc.buildWatch, "watch", "w", false, "watch filesystem for changes and recreate as needed")
+
+	cc.cmd.Flags().Bool("renderToMemory", false, "render to memory (only useful for benchmark testing)")
+
+	// Set bash-completion
+	_ = cc.cmd.PersistentFlags().SetAnnotation("logFile", cobra.BashCompFilenameExt, []string{})
+
+	cc.cmd.SetGlobalNormalizationFunc(helpers.NormalizeHugoFlags)
+	cc.cmd.SilenceUsage = true
+
+	return cc
+}
+
+type hugoBuilderCommon struct {
+	source  string
+	baseURL string
+
+	buildWatch bool
+
+	gc bool
+
+	// TODO(bep) var vs string
+	logging    bool
+	verbose    bool
+	verboseLog bool
+	debug      bool
+	quiet      bool
+
+	cfgFile string
+	logFile string
+}
+
+func (cc *hugoBuilderCommon) handleFlags(cmd *cobra.Command) {
+	cmd.Flags().Bool("cleanDestinationDir", false, "remove files from destination not found in static directories")
+	cmd.Flags().BoolP("buildDrafts", "D", false, "include content marked as draft")
+	cmd.Flags().BoolP("buildFuture", "F", false, "include content with publishdate in the future")
+	cmd.Flags().BoolP("buildExpired", "E", false, "include expired content")
+	cmd.Flags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")
+	cmd.Flags().StringP("contentDir", "c", "", "filesystem path to content directory")
+	cmd.Flags().StringP("layoutDir", "l", "", "filesystem path to layout directory")
+	cmd.Flags().StringP("cacheDir", "", "", "filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/")
+	cmd.Flags().BoolP("ignoreCache", "", false, "ignores the cache directory")
+	cmd.Flags().StringP("destination", "d", "", "filesystem path to write files to")
+	cmd.Flags().StringP("theme", "t", "", "theme to use (located in /themes/THEMENAME/)")
+	cmd.Flags().StringP("themesDir", "", "", "filesystem path to themes directory")
+	cmd.Flags().StringVarP(&cc.baseURL, "baseURL", "b", "", "hostname (and path) to the root, e.g. http://spf13.com/")
+	cmd.Flags().Bool("enableGitInfo", false, "add Git revision, date and author info to the pages")
+	cmd.Flags().BoolVar(&cc.gc, "gc", false, "enable to run some cleanup tasks (remove unused cache files) after the build")
+
+	cmd.Flags().BoolVar(&nitro.AnalysisOn, "stepAnalysis", false, "display memory and timing of different steps of the program")
+	cmd.Flags().Bool("templateMetrics", false, "display metrics about template executions")
+	cmd.Flags().Bool("templateMetricsHints", false, "calculate some improvement hints when combined with --templateMetrics")
+	cmd.Flags().BoolP("forceSyncStatic", "", false, "copy all files when static is changed.")
+	cmd.Flags().BoolP("noTimes", "", false, "don't sync modification time of files")
+	cmd.Flags().BoolP("noChmod", "", false, "don't sync permission mode of files")
+	cmd.Flags().BoolP("i18n-warnings", "", false, "print missing translations")
+
+	cmd.Flags().StringSlice("disableKinds", []string{}, "disable different kind of pages (home, RSS etc.)")
+
+	cmd.Flags().Bool("minify", false, "minify any supported output format (HTML, XML etc.)")
+
+	// Set bash-completion.
+	// Each flag must first be defined before using the SetAnnotation() call.
+	_ = cmd.Flags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{})
+	_ = cmd.Flags().SetAnnotation("cacheDir", cobra.BashCompSubdirsInDir, []string{})
+	_ = cmd.Flags().SetAnnotation("destination", cobra.BashCompSubdirsInDir, []string{})
+	_ = cmd.Flags().SetAnnotation("theme", cobra.BashCompSubdirsInDir, []string{"themes"})
+}
+
+func checkErr(logger *loggers.Logger, err error, s ...string) {
+	if err == nil {
+		return
+	}
+	if len(s) == 0 {
+		logger.CRITICAL.Println(err)
+		return
+	}
+	for _, message := range s {
+		logger.ERROR.Println(message)
+	}
+	logger.ERROR.Println(err)
+}

commands/commands_test.go

@@ -0,0 +1,252 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package commands
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestExecute(t *testing.T) {
+
+	assert := require.New(t)
+
+	dir, err := createSimpleTestSite(t)
+	assert.NoError(err)
+
+	defer func() {
+		os.RemoveAll(dir)
+	}()
+
+	resp := Execute([]string{"-s=" + dir})
+	assert.NoError(resp.Err)
+	result := resp.Result
+	assert.True(len(result.Sites) == 1)
+	assert.True(len(result.Sites[0].RegularPages) == 1)
+}
+
+func TestCommandsPersistentFlags(t *testing.T) {
+	assert := require.New(t)
+
+	noOpRunE := func(cmd *cobra.Command, args []string) error {
+		return nil
+	}
+
+	tests := []struct {
+		args  []string
+		check func(command []cmder)
+	}{{[]string{"server",
+		"--config=myconfig.toml",
+		"--contentDir=mycontent",
+		"--disableKinds=page,home",
+		"--layoutDir=mylayouts",
+		"--theme=mytheme",
+		"--gc",
+		"--themesDir=mythemes",
+		"--cleanDestinationDir",
+		"--navigateToChanged",
+		"--disableLiveReload",
+		"--noHTTPCache",
+		"--i18n-warnings",
+		"--destination=/tmp/mydestination",
+		"-b=https://example.com/b/",
+		"--port=1366",
+		"--renderToDisk",
+		"--source=mysource",
+	}, func(commands []cmder) {
+		var sc *serverCmd
+		for _, command := range commands {
+			if b, ok := command.(commandsBuilderGetter); ok {
+				v := b.getCommandsBuilder().hugoBuilderCommon
+				assert.Equal("myconfig.toml", v.cfgFile)
+				assert.Equal("mysource", v.source)
+				assert.Equal("https://example.com/b/", v.baseURL)
+			}
+
+			if srvCmd, ok := command.(*serverCmd); ok {
+				sc = srvCmd
+			}
+		}
+
+		assert.NotNil(sc)
+		assert.True(sc.navigateToChanged)
+		assert.True(sc.disableLiveReload)
+		assert.True(sc.noHTTPCache)
+		assert.True(sc.renderToDisk)
+		assert.Equal(1366, sc.serverPort)
+
+		cfg := viper.New()
+		sc.flagsToConfig(cfg)
+		assert.Equal("/tmp/mydestination", cfg.GetString("publishDir"))
+		assert.Equal("mycontent", cfg.GetString("contentDir"))
+		assert.Equal("mylayouts", cfg.GetString("layoutDir"))
+		assert.Equal("mytheme", cfg.GetString("theme"))
+		assert.Equal("mythemes", cfg.GetString("themesDir"))
+		assert.Equal("https://example.com/b/", cfg.GetString("baseURL"))
+
+		assert.Equal([]string{"page", "home"}, cfg.Get("disableKinds"))
+
+		assert.True(cfg.GetBool("gc"))
+
+		// The flag is named i18n-warnings
+		assert.True(cfg.GetBool("logI18nWarnings"))
+
+	}}}
+
+	for _, test := range tests {
+		b := newCommandsBuilder()
+		root := b.addAll().build()
+
+		for _, c := range b.commands {
+			if c.getCommand() == nil {
+				continue
+			}
+			// We are only intereseted in the flag handling here.
+			c.getCommand().RunE = noOpRunE
+		}
+		rootCmd := root.getCommand()
+		rootCmd.SetArgs(test.args)
+		assert.NoError(rootCmd.Execute())
+		test.check(b.commands)
+	}
+
+}
+
+func TestCommandsExecute(t *testing.T) {
+
+	assert := require.New(t)
+
+	dir, err := createSimpleTestSite(t)
+	assert.NoError(err)
+
+	dirOut, err := ioutil.TempDir("", "hugo-cli-out")
+	assert.NoError(err)
+
+	defer func() {
+		os.RemoveAll(dir)
+		os.RemoveAll(dirOut)
+	}()
+
+	sourceFlag := fmt.Sprintf("-s=%s", dir)
+
+	tests := []struct {
+		commands           []string
+		flags              []string
+		expectErrToContain string
+	}{
+		// TODO(bep) permission issue on my OSX? "operation not permitted" {[]string{"check", "ulimit"}, nil, false},
+		{[]string{"env"}, nil, ""},
+		{[]string{"version"}, nil, ""},
+		// no args = hugo build
+		{nil, []string{sourceFlag}, ""},
+		{nil, []string{sourceFlag, "--renderToMemory"}, ""},
+		{[]string{"config"}, []string{sourceFlag}, ""},
+		{[]string{"benchmark"}, []string{sourceFlag, "-n=1"}, ""},
+		{[]string{"convert", "toTOML"}, []string{sourceFlag, "-o=" + filepath.Join(dirOut, "toml")}, ""},
+		{[]string{"convert", "toYAML"}, []string{sourceFlag, "-o=" + filepath.Join(dirOut, "yaml")}, ""},
+		{[]string{"convert", "toJSON"}, []string{sourceFlag, "-o=" + filepath.Join(dirOut, "json")}, ""},
+		{[]string{"gen", "autocomplete"}, []string{"--completionfile=" + filepath.Join(dirOut, "autocomplete.txt")}, ""},
+		{[]string{"gen", "chromastyles"}, []string{"--style=manni"}, ""},
+		{[]string{"gen", "doc"}, []string{"--dir=" + filepath.Join(dirOut, "doc")}, ""},
+		{[]string{"gen", "man"}, []string{"--dir=" + filepath.Join(dirOut, "man")}, ""},
+		{[]string{"list", "drafts"}, []string{sourceFlag}, ""},
+		{[]string{"list", "expired"}, []string{sourceFlag}, ""},
+		{[]string{"list", "future"}, []string{sourceFlag}, ""},
+		{[]string{"new", "new-page.md"}, []string{sourceFlag}, ""},
+		{[]string{"new", "site", filepath.Join(dirOut, "new-site")}, nil, ""},
+		{[]string{"unknowncommand"}, nil, "unknown command"},
+		// TODO(bep) cli refactor fix https://github.com/gohugoio/hugo/issues/4450
+		//{[]string{"new", "theme", filepath.Join(dirOut, "new-theme")}, nil,false},
+	}
+
+	for _, test := range tests {
+
+		hugoCmd := newCommandsBuilder().addAll().build().getCommand()
+		test.flags = append(test.flags, "--quiet")
+		hugoCmd.SetArgs(append(test.commands, test.flags...))
+
+		// TODO(bep) capture output and add some simple asserts
+		// TODO(bep) misspelled subcommands does not return an error. We should investigate this
+		// but before that, check for "Error: unknown command".
+
+		_, err := hugoCmd.ExecuteC()
+		if test.expectErrToContain != "" {
+			assert.Error(err, fmt.Sprintf("%v", test.commands))
+			assert.Contains(err.Error(), test.expectErrToContain)
+		} else {
+			assert.NoError(err, fmt.Sprintf("%v", test.commands))
+		}
+
+	}
+
+}
+
+func createSimpleTestSite(t *testing.T) (string, error) {
+	d, e := ioutil.TempDir("", "hugo-cli")
+	if e != nil {
+		return "", e
+	}
+
+	// Just the basic. These are for CLI tests, not site testing.
+	writeFile(t, filepath.Join(d, "config.toml"), `
+
+baseURL = "https://example.org"
+title = "Hugo Commands"
+
+`)
+
+	writeFile(t, filepath.Join(d, "content", "p1.md"), `
+---
+title: "P1"
+weight: 1
+---
+
+Content
+
+`)
+
+	writeFile(t, filepath.Join(d, "layouts", "_default", "single.html"), `
+
+Single: {{ .Title }}
+
+`)
+
+	writeFile(t, filepath.Join(d, "layouts", "_default", "list.html"), `
+
+List: {{ .Title }}
+
+`)
+
+	return d, nil
+
+}
+
+func writeFile(t *testing.T, filename, content string) {
+	must(t, os.MkdirAll(filepath.Dir(filename), os.FileMode(0755)))
+	must(t, ioutil.WriteFile(filename, []byte(content), os.FileMode(0755)))
+}
+
+func must(t *testing.T, err error) {
+	if err != nil {
+		t.Fatal(err)
+	}
+}

commands/config.go

@@ -22,18 +22,29 @@ import (
 	"github.com/spf13/viper"
 )
 
-var configCmd = &cobra.Command{
-	Use:   "config",
-	Short: "Print the site configuration",
-	Long:  `Print the site configuration, both default and custom settings.`,
+var _ cmder = (*configCmd)(nil)
+
+type configCmd struct {
+	hugoBuilderCommon
+	*baseCmd
 }
 
-func init() {
-	configCmd.RunE = printConfig
+func newConfigCmd() *configCmd {
+	cc := &configCmd{}
+	cc.baseCmd = newBaseCmd(&cobra.Command{
+		Use:   "config",
+		Short: "Print the site configuration",
+		Long:  `Print the site configuration, both default and custom settings.`,
+		RunE:  cc.printConfig,
+	})
+
+	cc.cmd.Flags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")
+
+	return cc
 }
 
-func printConfig(cmd *cobra.Command, args []string) error {
-	cfg, err := InitializeConfig(false, nil, configCmd)
+func (c *configCmd) printConfig(cmd *cobra.Command, args []string) error {
+	cfg, err := initializeConfig(true, false, &c.hugoBuilderCommon, c, nil)
 
 	if err != nil {
 		return err

commands/convert.go

@@ -14,82 +14,105 @@
 package commands
 
 import (
+	"bytes"
 	"fmt"
+	"io"
+	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/hugofs"
+
+	"github.com/gohugoio/hugo/helpers"
+
+	"github.com/gohugoio/hugo/parser"
+	"github.com/gohugoio/hugo/parser/metadecoders"
+	"github.com/gohugoio/hugo/parser/pageparser"
+
 	src "github.com/gohugoio/hugo/source"
+	"github.com/pkg/errors"
 
 	"github.com/gohugoio/hugo/hugolib"
 
 	"path/filepath"
 
-	"github.com/gohugoio/hugo/parser"
-	"github.com/spf13/cast"
 	"github.com/spf13/cobra"
 )
 
-var outputDir string
-var unsafe bool
+var (
+	_ cmder = (*convertCmd)(nil)
+)
 
-var convertCmd = &cobra.Command{
-	Use:   "convert",
-	Short: "Convert your content to different formats",
-	Long: `Convert your content (e.g. front matter) to different formats.
+type convertCmd struct {
+	hugoBuilderCommon
+
+	outputDir string
+	unsafe    bool
+
+	*baseCmd
+}
+
+func newConvertCmd() *convertCmd {
+	cc := &convertCmd{}
+
+	cc.baseCmd = newBaseCmd(&cobra.Command{
+		Use:   "convert",
+		Short: "Convert your content to different formats",
+		Long: `Convert your content (e.g. front matter) to different formats.
 
 See convert's subcommands toJSON, toTOML and toYAML for more information.`,
-	RunE: nil,
-}
+		RunE: nil,
+	})
 
-var toJSONCmd = &cobra.Command{
-	Use:   "toJSON",
-	Short: "Convert front matter to JSON",
-	Long: `toJSON converts all front matter in the content directory
+	cc.cmd.AddCommand(
+		&cobra.Command{
+			Use:   "toJSON",
+			Short: "Convert front matter to JSON",
+			Long: `toJSON converts all front matter in the content directory
 to use JSON for the front matter.`,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return convertContents(rune([]byte(parser.JSONLead)[0]))
-	},
-}
-
-var toTOMLCmd = &cobra.Command{
-	Use:   "toTOML",
-	Short: "Convert front matter to TOML",
-	Long: `toTOML converts all front matter in the content directory
+			RunE: func(cmd *cobra.Command, args []string) error {
+				return cc.convertContents(metadecoders.JSON)
+			},
+		},
+		&cobra.Command{
+			Use:   "toTOML",
+			Short: "Convert front matter to TOML",
+			Long: `toTOML converts all front matter in the content directory
 to use TOML for the front matter.`,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return convertContents(rune([]byte(parser.TOMLLead)[0]))
-	},
-}
-
-var toYAMLCmd = &cobra.Command{
-	Use:   "toYAML",
-	Short: "Convert front matter to YAML",
-	Long: `toYAML converts all front matter in the content directory
+			RunE: func(cmd *cobra.Command, args []string) error {
+				return cc.convertContents(metadecoders.TOML)
+			},
+		},
+		&cobra.Command{
+			Use:   "toYAML",
+			Short: "Convert front matter to YAML",
+			Long: `toYAML converts all front matter in the content directory
 to use YAML for the front matter.`,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return convertContents(rune([]byte(parser.YAMLLead)[0]))
-	},
+			RunE: func(cmd *cobra.Command, args []string) error {
+				return cc.convertContents(metadecoders.YAML)
+			},
+		},
+	)
+
+	cc.cmd.PersistentFlags().StringVarP(&cc.outputDir, "output", "o", "", "filesystem path to write files to")
+	cc.cmd.PersistentFlags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")
+	cc.cmd.PersistentFlags().BoolVar(&cc.unsafe, "unsafe", false, "enable less safe operations, please backup first")
+	cc.cmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{})
+
+	return cc
 }
 
-func init() {
-	convertCmd.AddCommand(toJSONCmd)
-	convertCmd.AddCommand(toTOMLCmd)
-	convertCmd.AddCommand(toYAMLCmd)
-	convertCmd.PersistentFlags().StringVarP(&outputDir, "output", "o", "", "filesystem path to write files to")
-	convertCmd.PersistentFlags().StringVarP(&source, "source", "s", "", "filesystem path to read files relative from")
-	convertCmd.PersistentFlags().BoolVar(&unsafe, "unsafe", false, "enable less safe operations, please backup first")
-	convertCmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{})
-}
-
-func convertContents(mark rune) error {
-	if outputDir == "" && !unsafe {
+func (cc *convertCmd) convertContents(format metadecoders.Format) error {
+	if cc.outputDir == "" && !cc.unsafe {
 		return newUserError("Unsafe operation not allowed, use --unsafe or set a different output path")
 	}
 
-	c, err := InitializeConfig(false, nil)
+	c, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, nil)
 	if err != nil {
 		return err
 	}
 
+	c.Cfg.Set("buildDrafts", true)
+
 	h, err := hugolib.NewHugoSites(*c.DepsCfg)
 	if err != nil {
 		return err
@@ -103,17 +126,17 @@ func convertContents(mark rune) error {
 
 	site.Log.FEEDBACK.Println("processing", len(site.AllPages), "content files")
 	for _, p := range site.AllPages {
-		if err := convertAndSavePage(p, site, mark); err != nil {
+		if err := cc.convertAndSavePage(p, site, format); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func convertAndSavePage(p *hugolib.Page, site *hugolib.Site, mark rune) error {
+func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, targetFormat metadecoders.Format) error {
 	// The resources are not in .Site.AllPages.
 	for _, r := range p.Resources.ByType("page") {
-		if err := convertAndSavePage(r.(*hugolib.Page), site, mark); err != nil {
+		if err := cc.convertAndSavePage(r.(*hugolib.Page), site, targetFormat); err != nil {
 			return err
 		}
 	}
@@ -123,61 +146,104 @@ func convertAndSavePage(p *hugolib.Page, site *hugolib.Site, mark rune) error {
 		return nil
 	}
 
+	errMsg := fmt.Errorf("Error processing file %q", p.Path())
+
 	site.Log.INFO.Println("Attempting to convert", p.LogicalName())
-	newPage, err := site.NewPage(p.LogicalName())
-	if err != nil {
-		return err
-	}
 
 	f, _ := p.File.(src.ReadableFile)
 	file, err := f.Open()
 	if err != nil {
-		site.Log.ERROR.Println("Error reading file:", p.Path())
+		site.Log.ERROR.Println(errMsg)
 		file.Close()
 		return nil
 	}
 
-	psr, err := parser.ReadFrom(file)
+	pf, err := parseContentFile(file)
 	if err != nil {
-		site.Log.ERROR.Println("Error processing file:", p.Path())
+		site.Log.ERROR.Println(errMsg)
 		file.Close()
 		return err
 	}
 
 	file.Close()
 
-	metadata, err := psr.Metadata()
+	// better handling of dates in formats that don't have support for them
+	if pf.frontMatterFormat == metadecoders.JSON || pf.frontMatterFormat == metadecoders.YAML || pf.frontMatterFormat == metadecoders.TOML {
+		for k, v := range pf.frontMatter {
+			switch vv := v.(type) {
+			case time.Time:
+				pf.frontMatter[k] = vv.Format(time.RFC3339)
+			}
+		}
+	}
+
+	var newContent bytes.Buffer
+	err = parser.InterfaceToFrontMatter(pf.frontMatter, targetFormat, &newContent)
 	if err != nil {
-		site.Log.ERROR.Println("Error processing file:", p.Path())
+		site.Log.ERROR.Println(errMsg)
 		return err
 	}
 
-	// better handling of dates in formats that don't have support for them
-	if mark == parser.FormatToLeadRune("json") || mark == parser.FormatToLeadRune("yaml") || mark == parser.FormatToLeadRune("toml") {
-		newMetadata := cast.ToStringMap(metadata)
-		for k, v := range newMetadata {
-			switch vv := v.(type) {
-			case time.Time:
-				newMetadata[k] = vv.Format(time.RFC3339)
-			}
-		}
-		metadata = newMetadata
-	}
-
-	newPage.SetSourceContent(psr.Content())
-	if err = newPage.SetSourceMetaData(metadata, mark); err != nil {
-		site.Log.ERROR.Printf("Failed to set source metadata for file %q: %s. For more info see For more info see https://github.com/gohugoio/hugo/issues/2458", newPage.FullFilePath(), err)
-		return nil
-	}
+	newContent.Write(pf.content)
 
 	newFilename := p.Filename()
-	if outputDir != "" {
-		newFilename = filepath.Join(outputDir, p.Dir(), newPage.LogicalName())
+
+	if cc.outputDir != "" {
+		contentDir := strings.TrimSuffix(newFilename, p.Path())
+		contentDir = filepath.Base(contentDir)
+
+		newFilename = filepath.Join(cc.outputDir, contentDir, p.Path())
 	}
 
-	if err = newPage.SaveSourceAs(newFilename); err != nil {
-		return fmt.Errorf("Failed to save file %q: %s", newFilename, err)
+	fs := hugofs.Os
+	if err := helpers.WriteToDisk(newFilename, &newContent, fs); err != nil {
+		return errors.Wrapf(err, "Failed to save file %q:", newFilename)
 	}
 
 	return nil
 }
+
+type parsedFile struct {
+	frontMatterFormat metadecoders.Format
+	frontMatterSource []byte
+	frontMatter       map[string]interface{}
+
+	// Everything after Front Matter
+	content []byte
+}
+
+func parseContentFile(r io.Reader) (parsedFile, error) {
+	var pf parsedFile
+
+	psr, err := pageparser.Parse(r)
+	if err != nil {
+		return pf, err
+	}
+
+	iter := psr.Iterator()
+
+	walkFn := func(item pageparser.Item) bool {
+		if pf.frontMatterSource != nil {
+			// The rest is content.
+			pf.content = psr.Input()[item.Pos:]
+			// Done
+			return false
+		} else if item.IsFrontMatter() {
+			pf.frontMatterFormat = metadecoders.FormatFromFrontMatterType(item.Type)
+			pf.frontMatterSource = item.Val
+		}
+		return true
+
+	}
+
+	iter.PeekWalk(walkFn)
+
+	metadata, err := metadecoders.UnmarshalToMap(pf.frontMatterSource, pf.frontMatterFormat)
+	if err != nil {
+		return pf, err
+	}
+	pf.frontMatter = metadata
+
+	return pf, nil
+
+}

commands/env.go

@@ -20,16 +20,25 @@ import (
 	jww "github.com/spf13/jwalterweatherman"
 )
 
-var envCmd = &cobra.Command{
-	Use:   "env",
-	Short: "Print Hugo version and environment info",
-	Long:  `Print Hugo version and environment info. This is useful in Hugo bug reports.`,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		printHugoVersion()
-		jww.FEEDBACK.Printf("GOOS=%q\n", runtime.GOOS)
-		jww.FEEDBACK.Printf("GOARCH=%q\n", runtime.GOARCH)
-		jww.FEEDBACK.Printf("GOVERSION=%q\n", runtime.Version())
+var _ cmder = (*envCmd)(nil)
 
-		return nil
-	},
+type envCmd struct {
+	*baseCmd
+}
+
+func newEnvCmd() *envCmd {
+	return &envCmd{baseCmd: newBaseCmd(&cobra.Command{
+		Use:   "env",
+		Short: "Print Hugo version and environment info",
+		Long:  `Print Hugo version and environment info. This is useful in Hugo bug reports.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			printHugoVersion()
+			jww.FEEDBACK.Printf("GOOS=%q\n", runtime.GOOS)
+			jww.FEEDBACK.Printf("GOARCH=%q\n", runtime.GOARCH)
+			jww.FEEDBACK.Printf("GOVERSION=%q\n", runtime.Version())
+
+			return nil
+		},
+	}),
+	}
 }

commands/gen.go

@@ -17,7 +17,25 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var genCmd = &cobra.Command{
-	Use:   "gen",
-	Short: "A collection of several useful generators.",
+var _ cmder = (*genCmd)(nil)
+
+type genCmd struct {
+	*baseCmd
+}
+
+func newGenCmd() *genCmd {
+	cc := &genCmd{}
+	cc.baseCmd = newBaseCmd(&cobra.Command{
+		Use:   "gen",
+		Short: "A collection of several useful generators.",
+	})
+
+	cc.cmd.AddCommand(
+		newGenautocompleteCmd().getCommand(),
+		newGenDocCmd().getCommand(),
+		newGenManCmd().getCommand(),
+		createGenDocsHelper().getCommand(),
+		createGenChromaStyles().getCommand())
+
+	return cc
 }

commands/genautocomplete.go

@@ -18,15 +18,24 @@ import (
 	jww "github.com/spf13/jwalterweatherman"
 )
 
-var autocompleteTarget string
+var _ cmder = (*genautocompleteCmd)(nil)
 
-// bash for now (zsh and others will come)
-var autocompleteType string
+type genautocompleteCmd struct {
+	autocompleteTarget string
 
-var genautocompleteCmd = &cobra.Command{
-	Use:   "autocomplete",
-	Short: "Generate shell autocompletion script for Hugo",
-	Long: `Generates a shell autocompletion script for Hugo.
+	// bash for now (zsh and others will come)
+	autocompleteType string
+
+	*baseCmd
+}
+
+func newGenautocompleteCmd() *genautocompleteCmd {
+	cc := &genautocompleteCmd{}
+
+	cc.baseCmd = newBaseCmd(&cobra.Command{
+		Use:   "autocomplete",
+		Short: "Generate shell autocompletion script for Hugo",
+		Long: `Generates a shell autocompletion script for Hugo.
 
 NOTE: The current version supports Bash only.
       This should work for *nix systems with Bash installed.
@@ -44,27 +53,28 @@ or just source them in directly:
 
 	$ . /etc/bash_completion`,
 
-	RunE: func(cmd *cobra.Command, args []string) error {
-		if autocompleteType != "bash" {
-			return newUserError("Only Bash is supported for now")
-		}
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if cc.autocompleteType != "bash" {
+				return newUserError("Only Bash is supported for now")
+			}
 
-		err := cmd.Root().GenBashCompletionFile(autocompleteTarget)
+			err := cmd.Root().GenBashCompletionFile(cc.autocompleteTarget)
 
-		if err != nil {
-			return err
-		}
+			if err != nil {
+				return err
+			}
 
-		jww.FEEDBACK.Println("Bash completion file for Hugo saved to", autocompleteTarget)
+			jww.FEEDBACK.Println("Bash completion file for Hugo saved to", cc.autocompleteTarget)
 
-		return nil
-	},
-}
+			return nil
+		},
+	})
 
-func init() {
-	genautocompleteCmd.PersistentFlags().StringVarP(&autocompleteTarget, "completionfile", "", "/etc/bash_completion.d/hugo.sh", "autocompletion file")
-	genautocompleteCmd.PersistentFlags().StringVarP(&autocompleteType, "type", "", "bash", "autocompletion type (currently only bash supported)")
+	cc.cmd.PersistentFlags().StringVarP(&cc.autocompleteTarget, "completionfile", "", "/etc/bash_completion.d/hugo.sh", "autocompletion file")
+	cc.cmd.PersistentFlags().StringVarP(&cc.autocompleteType, "type", "", "bash", "autocompletion type (currently only bash supported)")
 
 	// For bash-completion
-	genautocompleteCmd.PersistentFlags().SetAnnotation("completionfile", cobra.BashCompFilenameExt, []string{})
+	cc.cmd.PersistentFlags().SetAnnotation("completionfile", cobra.BashCompFilenameExt, []string{})
+
+	return cc
 }

commands/genchromastyles.go

@@ -22,23 +22,27 @@ import (
 	"github.com/spf13/cobra"
 )
 
+var (
+	_ cmder = (*genChromaStyles)(nil)
+)
+
 type genChromaStyles struct {
 	style          string
 	highlightStyle string
 	linesStyle     string
-	cmd            *cobra.Command
+	*baseCmd
 }
 
 // TODO(bep) highlight
 func createGenChromaStyles() *genChromaStyles {
 	g := &genChromaStyles{
-		cmd: &cobra.Command{
+		baseCmd: newBaseCmd(&cobra.Command{
 			Use:   "chromastyles",
 			Short: "Generate CSS stylesheet for the Chroma code highlighter",
 			Long: `Generate CSS stylesheet for the Chroma code highlighter for a given style. This stylesheet is needed if pygmentsUseClasses is enabled in config.
 
 See https://help.farbox.com/pygments.html for preview of available styles`,
-		},
+		}),
 	}
 
 	g.cmd.RunE = func(cmd *cobra.Command, args []string) error {

commands/gendoc.go

@@ -27,7 +27,15 @@ import (
 	jww "github.com/spf13/jwalterweatherman"
 )
 
-const gendocFrontmatterTemplate = `---
+var _ cmder = (*genDocCmd)(nil)
+
+type genDocCmd struct {
+	gendocdir string
+	*baseCmd
+}
+
+func newGenDocCmd() *genDocCmd {
+	const gendocFrontmatterTemplate = `---
 date: %s
 title: "%s"
 slug: %s
@@ -35,11 +43,12 @@ url: %s
 ---
 `
 
-var gendocdir string
-var gendocCmd = &cobra.Command{
-	Use:   "doc",
-	Short: "Generate Markdown documentation for the Hugo CLI.",
-	Long: `Generate Markdown documentation for the Hugo CLI.
+	cc := &genDocCmd{}
+
+	cc.baseCmd = newBaseCmd(&cobra.Command{
+		Use:   "doc",
+		Short: "Generate Markdown documentation for the Hugo CLI.",
+		Long: `Generate Markdown documentation for the Hugo CLI.
 
 This command is, mostly, used to create up-to-date documentation
 of Hugo's command-line interface for http://gohugo.io/.
@@ -47,40 +56,41 @@ of Hugo's command-line interface for http://gohugo.io/.
 It creates one Markdown file per command with front matter suitable
 for rendering in Hugo.`,
 
-	RunE: func(cmd *cobra.Command, args []string) error {
-		if !strings.HasSuffix(gendocdir, helpers.FilePathSeparator) {
-			gendocdir += helpers.FilePathSeparator
-		}
-		if found, _ := helpers.Exists(gendocdir, hugofs.Os); !found {
-			jww.FEEDBACK.Println("Directory", gendocdir, "does not exist, creating...")
-			if err := hugofs.Os.MkdirAll(gendocdir, 0777); err != nil {
-				return err
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if !strings.HasSuffix(cc.gendocdir, helpers.FilePathSeparator) {
+				cc.gendocdir += helpers.FilePathSeparator
+			}
+			if found, _ := helpers.Exists(cc.gendocdir, hugofs.Os); !found {
+				jww.FEEDBACK.Println("Directory", cc.gendocdir, "does not exist, creating...")
+				if err := hugofs.Os.MkdirAll(cc.gendocdir, 0777); err != nil {
+					return err
+				}
+			}
+			now := time.Now().Format("2006-01-02")
+			prepender := func(filename string) string {
+				name := filepath.Base(filename)
+				base := strings.TrimSuffix(name, path.Ext(name))
+				url := "/commands/" + strings.ToLower(base) + "/"
+				return fmt.Sprintf(gendocFrontmatterTemplate, now, strings.Replace(base, "_", " ", -1), base, url)
 			}
-		}
-		now := time.Now().Format(time.RFC3339)
-		prepender := func(filename string) string {
-			name := filepath.Base(filename)
-			base := strings.TrimSuffix(name, path.Ext(name))
-			url := "/commands/" + strings.ToLower(base) + "/"
-			return fmt.Sprintf(gendocFrontmatterTemplate, now, strings.Replace(base, "_", " ", -1), base, url)
-		}
 
-		linkHandler := func(name string) string {
-			base := strings.TrimSuffix(name, path.Ext(name))
-			return "/commands/" + strings.ToLower(base) + "/"
-		}
+			linkHandler := func(name string) string {
+				base := strings.TrimSuffix(name, path.Ext(name))
+				return "/commands/" + strings.ToLower(base) + "/"
+			}
 
-		jww.FEEDBACK.Println("Generating Hugo command-line documentation in", gendocdir, "...")
-		doc.GenMarkdownTreeCustom(cmd.Root(), gendocdir, prepender, linkHandler)
-		jww.FEEDBACK.Println("Done.")
+			jww.FEEDBACK.Println("Generating Hugo command-line documentation in", cc.gendocdir, "...")
+			doc.GenMarkdownTreeCustom(cmd.Root(), cc.gendocdir, prepender, linkHandler)
+			jww.FEEDBACK.Println("Done.")
 
-		return nil
-	},
-}
+			return nil
+		},
+	})
 
-func init() {
-	gendocCmd.PersistentFlags().StringVar(&gendocdir, "dir", "/tmp/hugodoc/", "the directory to write the doc.")
+	cc.cmd.PersistentFlags().StringVar(&cc.gendocdir, "dir", "/tmp/hugodoc/", "the directory to write the doc.")
 
 	// For bash-completion
-	gendocCmd.PersistentFlags().SetAnnotation("dir", cobra.BashCompSubdirsInDir, []string{})
+	cc.cmd.PersistentFlags().SetAnnotation("dir", cobra.BashCompSubdirsInDir, []string{})
+
+	return cc
 }

commands/gendocshelper.go

@@ -23,18 +23,22 @@ import (
 	"github.com/spf13/cobra"
 )
 
+var (
+	_ cmder = (*genDocsHelper)(nil)
+)
+
 type genDocsHelper struct {
 	target string
-	cmd    *cobra.Command
+	*baseCmd
 }
 
 func createGenDocsHelper() *genDocsHelper {
 	g := &genDocsHelper{
-		cmd: &cobra.Command{
+		baseCmd: newBaseCmd(&cobra.Command{
 			Use:    "docshelper",
 			Short:  "Generate some data files for the Hugo docs.",
 			Hidden: true,
-		},
+		}),
 	}
 
 	g.cmd.RunE = func(cmd *cobra.Command, args []string) error {

commands/genman.go

@@ -24,43 +24,53 @@ import (
 	jww "github.com/spf13/jwalterweatherman"
 )
 
-var genmandir string
-var genmanCmd = &cobra.Command{
-	Use:   "man",
-	Short: "Generate man pages for the Hugo CLI",
-	Long: `This command automatically generates up-to-date man pages of Hugo's
+var _ cmder = (*genManCmd)(nil)
+
+type genManCmd struct {
+	genmandir string
+	*baseCmd
+}
+
+func newGenManCmd() *genManCmd {
+	cc := &genManCmd{}
+
+	cc.baseCmd = newBaseCmd(&cobra.Command{
+		Use:   "man",
+		Short: "Generate man pages for the Hugo CLI",
+		Long: `This command automatically generates up-to-date man pages of Hugo's
 command-line interface.  By default, it creates the man page files
 in the "man" directory under the current directory.`,
 
-	RunE: func(cmd *cobra.Command, args []string) error {
-		header := &doc.GenManHeader{
-			Section: "1",
-			Manual:  "Hugo Manual",
-			Source:  fmt.Sprintf("Hugo %s", helpers.CurrentHugoVersion),
-		}
-		if !strings.HasSuffix(genmandir, helpers.FilePathSeparator) {
-			genmandir += helpers.FilePathSeparator
-		}
-		if found, _ := helpers.Exists(genmandir, hugofs.Os); !found {
-			jww.FEEDBACK.Println("Directory", genmandir, "does not exist, creating...")
-			if err := hugofs.Os.MkdirAll(genmandir, 0777); err != nil {
-				return err
+		RunE: func(cmd *cobra.Command, args []string) error {
+			header := &doc.GenManHeader{
+				Section: "1",
+				Manual:  "Hugo Manual",
+				Source:  fmt.Sprintf("Hugo %s", helpers.CurrentHugoVersion),
 			}
-		}
-		cmd.Root().DisableAutoGenTag = true
+			if !strings.HasSuffix(cc.genmandir, helpers.FilePathSeparator) {
+				cc.genmandir += helpers.FilePathSeparator
+			}
+			if found, _ := helpers.Exists(cc.genmandir, hugofs.Os); !found {
+				jww.FEEDBACK.Println("Directory", cc.genmandir, "does not exist, creating...")
+				if err := hugofs.Os.MkdirAll(cc.genmandir, 0777); err != nil {
+					return err
+				}
+			}
+			cmd.Root().DisableAutoGenTag = true
 
-		jww.FEEDBACK.Println("Generating Hugo man pages in", genmandir, "...")
-		doc.GenManTree(cmd.Root(), header, genmandir)
+			jww.FEEDBACK.Println("Generating Hugo man pages in", cc.genmandir, "...")
+			doc.GenManTree(cmd.Root(), header, cc.genmandir)
 
-		jww.FEEDBACK.Println("Done.")
+			jww.FEEDBACK.Println("Done.")
 
-		return nil
-	},
-}
+			return nil
+		},
+	})
 
-func init() {
-	genmanCmd.PersistentFlags().StringVar(&genmandir, "dir", "man/", "the directory to write the man pages.")
+	cc.cmd.PersistentFlags().StringVar(&cc.genmandir, "dir", "man/", "the directory to write the man pages.")
 
 	// For bash-completion
-	genmanCmd.PersistentFlags().SetAnnotation("dir", cobra.BashCompSubdirsInDir, []string{})
+	cc.cmd.PersistentFlags().SetAnnotation("dir", cobra.BashCompSubdirsInDir, []string{})
+
+	return cc
 }

commands/helpers.go

@@ -0,0 +1,79 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package commands defines and implements command-line commands and flags
+// used by Hugo. Commands and flags are implemented using Cobra.
+package commands
+
+import (
+	"fmt"
+	"regexp"
+
+	"github.com/gohugoio/hugo/config"
+	"github.com/spf13/cobra"
+)
+
+const (
+	ansiEsc    = "\u001B"
+	clearLine  = "\r\033[K"
+	hideCursor = ansiEsc + "[?25l"
+	showCursor = ansiEsc + "[?25h"
+)
+
+type flagsToConfigHandler interface {
+	flagsToConfig(cfg config.Provider)
+}
+
+type cmder interface {
+	flagsToConfigHandler
+	getCommand() *cobra.Command
+}
+
+// commandError is an error used to signal different error situations in command handling.
+type commandError struct {
+	s         string
+	userError bool
+}
+
+func (c commandError) Error() string {
+	return c.s
+}
+
+func (c commandError) isUserError() bool {
+	return c.userError
+}
+
+func newUserError(a ...interface{}) commandError {
+	return commandError{s: fmt.Sprintln(a...), userError: true}
+}
+
+func newSystemError(a ...interface{}) commandError {
+	return commandError{s: fmt.Sprintln(a...), userError: false}
+}
+
+func newSystemErrorF(format string, a ...interface{}) commandError {
+	return commandError{s: fmt.Sprintf(format, a...), userError: false}
+}
+
+// Catch some of the obvious user errors from Cobra.
+// We don't want to show the usage message for every error.
+// The below may be to generic. Time will show.
+var userErrorRegexp = regexp.MustCompile("argument|flag|shorthand")
+
+func isUserError(err error) bool {
+	if cErr, ok := err.(commandError); ok && cErr.isUserError() {
+		return true
+	}
+
+	return userErrorRegexp.MatchString(err.Error())
+}

commands/hugo_windows.go

@@ -23,5 +23,5 @@ func init() {
 
   You need to open cmd.exe and run Hugo from there.
   
-  Visit http://gohugo.io/ for more information.`
+  Visit https://gohugo.io/ for more information.`
 }

commands/import_jekyll.go

@@ -16,6 +16,7 @@ package commands
 import (
 	"bytes"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -25,6 +26,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/parser/metadecoders"
+
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/hugolib"
@@ -35,63 +38,72 @@ import (
 	jww "github.com/spf13/jwalterweatherman"
 )
 
-func init() {
-	importCmd.AddCommand(importJekyllCmd)
+var _ cmder = (*importCmd)(nil)
+
+type importCmd struct {
+	*baseCmd
 }
 
-var importCmd = &cobra.Command{
-	Use:   "import",
-	Short: "Import your site from others.",
-	Long: `Import your site from other web site generators like Jekyll.
+func newImportCmd() *importCmd {
+	cc := &importCmd{}
+
+	cc.baseCmd = newBaseCmd(&cobra.Command{
+		Use:   "import",
+		Short: "Import your site from others.",
+		Long: `Import your site from other web site generators like Jekyll.
 
 Import requires a subcommand, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.",
-	RunE: nil,
-}
+		RunE: nil,
+	})
 
-var importJekyllCmd = &cobra.Command{
-	Use:   "jekyll",
-	Short: "hugo import from Jekyll",
-	Long: `hugo import from Jekyll.
+	importJekyllCmd := &cobra.Command{
+		Use:   "jekyll",
+		Short: "hugo import from Jekyll",
+		Long: `hugo import from Jekyll.
 
 Import from Jekyll requires two paths, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.",
-	RunE: importFromJekyll,
-}
+		RunE: cc.importFromJekyll,
+	}
 
-func init() {
 	importJekyllCmd.Flags().Bool("force", false, "allow import into non-empty target directory")
+
+	cc.cmd.AddCommand(importJekyllCmd)
+
+	return cc
+
 }
 
-func importFromJekyll(cmd *cobra.Command, args []string) error {
+func (i *importCmd) importFromJekyll(cmd *cobra.Command, args []string) error {
 
 	if len(args) < 2 {
-		return newUserError(`Import from Jekyll requires two paths, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.")
+		return newUserError(`import from jekyll requires two paths, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.")
 	}
 
 	jekyllRoot, err := filepath.Abs(filepath.Clean(args[0]))
 	if err != nil {
-		return newUserError("Path error:", args[0])
+		return newUserError("path error:", args[0])
 	}
 
 	targetDir, err := filepath.Abs(filepath.Clean(args[1]))
 	if err != nil {
-		return newUserError("Path error:", args[1])
+		return newUserError("path error:", args[1])
 	}
 
 	jww.INFO.Println("Import Jekyll from:", jekyllRoot, "to:", targetDir)
 
 	if strings.HasPrefix(filepath.Dir(targetDir), jekyllRoot) {
-		return newUserError("Target path should not be inside the Jekyll root, aborting.")
+		return newUserError("abort: target path should not be inside the Jekyll root")
 	}
 
 	forceImport, _ := cmd.Flags().GetBool("force")
 
 	fs := afero.NewOsFs()
-	jekyllPostDirs, hasAnyPost := getJekyllDirInfo(fs, jekyllRoot)
+	jekyllPostDirs, hasAnyPost := i.getJekyllDirInfo(fs, jekyllRoot)
 	if !hasAnyPost {
-		return errors.New("Your Jekyll root contains neither posts nor drafts, aborting.")
+		return errors.New("abort: jekyll root contains neither posts nor drafts")
 	}
 
-	site, err := createSiteFromJekyll(jekyllRoot, targetDir, jekyllPostDirs, forceImport)
+	site, err := i.createSiteFromJekyll(jekyllRoot, targetDir, jekyllPostDirs, forceImport)
 
 	if err != nil {
 		return newUserError(err)
@@ -111,7 +123,7 @@ func importFromJekyll(cmd *cobra.Command, args []string) error {
 
 		relPath, err := filepath.Rel(jekyllRoot, path)
 		if err != nil {
-			return newUserError("Get rel path error:", path)
+			return newUserError("get rel path error:", path)
 		}
 
 		relPath = filepath.ToSlash(relPath)
@@ -144,22 +156,17 @@ func importFromJekyll(cmd *cobra.Command, args []string) error {
 		"$ git clone https://github.com/spf13/herring-cove.git " + args[1] + "/themes/herring-cove")
 	jww.FEEDBACK.Println("$ cd " + args[1] + "\n$ hugo server --theme=herring-cove")
 
-	jww.FEEDBACK.Println("Congratulations!", fileCount, "post(s) imported!")
-	jww.FEEDBACK.Println("Now, start Hugo by yourself:\n" +
-		"$ git clone https://github.com/spf13/herring-cove.git " + args[1] + "/themes/herring-cove")
-	jww.FEEDBACK.Println("$ cd " + args[1] + "\n$ hugo server --theme=herring-cove")
-
 	return nil
 }
 
-func getJekyllDirInfo(fs afero.Fs, jekyllRoot string) (map[string]bool, bool) {
+func (i *importCmd) getJekyllDirInfo(fs afero.Fs, jekyllRoot string) (map[string]bool, bool) {
 	postDirs := make(map[string]bool)
 	hasAnyPost := false
 	if entries, err := ioutil.ReadDir(jekyllRoot); err == nil {
 		for _, entry := range entries {
 			if entry.IsDir() {
 				subDir := filepath.Join(jekyllRoot, entry.Name())
-				if isPostDir, hasAnyPostInDir := retrieveJekyllPostDir(fs, subDir); isPostDir {
+				if isPostDir, hasAnyPostInDir := i.retrieveJekyllPostDir(fs, subDir); isPostDir {
 					postDirs[entry.Name()] = hasAnyPostInDir
 					if hasAnyPostInDir {
 						hasAnyPost = true
@@ -171,7 +178,7 @@ func getJekyllDirInfo(fs afero.Fs, jekyllRoot string) (map[string]bool, bool) {
 	return postDirs, hasAnyPost
 }
 
-func retrieveJekyllPostDir(fs afero.Fs, dir string) (bool, bool) {
+func (i *importCmd) retrieveJekyllPostDir(fs afero.Fs, dir string) (bool, bool) {
 	if strings.HasSuffix(dir, "_posts") || strings.HasSuffix(dir, "_drafts") {
 		isEmpty, _ := helpers.IsEmpty(dir, fs)
 		return true, !isEmpty
@@ -181,7 +188,7 @@ func retrieveJekyllPostDir(fs afero.Fs, dir string) (bool, bool) {
 		for _, entry := range entries {
 			if entry.IsDir() {
 				subDir := filepath.Join(dir, entry.Name())
-				if isPostDir, hasAnyPost := retrieveJekyllPostDir(fs, subDir); isPostDir {
+				if isPostDir, hasAnyPost := i.retrieveJekyllPostDir(fs, subDir); isPostDir {
 					return isPostDir, hasAnyPost
 				}
 			}
@@ -191,7 +198,7 @@ func retrieveJekyllPostDir(fs afero.Fs, dir string) (bool, bool) {
 	return false, true
 }
 
-func createSiteFromJekyll(jekyllRoot, targetDir string, jekyllPostDirs map[string]bool, force bool) (*hugolib.Site, error) {
+func (i *importCmd) createSiteFromJekyll(jekyllRoot, targetDir string, jekyllPostDirs map[string]bool, force bool) (*hugolib.Site, error) {
 	s, err := hugolib.NewSiteDefaultLang()
 	if err != nil {
 		return nil, err
@@ -200,17 +207,17 @@ func createSiteFromJekyll(jekyllRoot, targetDir string, jekyllPostDirs map[strin
 	fs := s.Fs.Source
 	if exists, _ := helpers.Exists(targetDir, fs); exists {
 		if isDir, _ := helpers.IsDir(targetDir, fs); !isDir {
-			return nil, errors.New("Target path \"" + targetDir + "\" already exists but not a directory")
+			return nil, errors.New("target path \"" + targetDir + "\" exists but is not a directory")
 		}
 
 		isEmpty, _ := helpers.IsEmpty(targetDir, fs)
 
 		if !isEmpty && !force {
-			return nil, errors.New("Target path \"" + targetDir + "\" already exists and is not empty")
+			return nil, errors.New("target path \"" + targetDir + "\" exists and is not empty")
 		}
 	}
 
-	jekyllConfig := loadJekyllConfig(fs, jekyllRoot)
+	jekyllConfig := i.loadJekyllConfig(fs, jekyllRoot)
 
 	mkdir(targetDir, "layouts")
 	mkdir(targetDir, "content")
@@ -219,14 +226,14 @@ func createSiteFromJekyll(jekyllRoot, targetDir string, jekyllPostDirs map[strin
 	mkdir(targetDir, "data")
 	mkdir(targetDir, "themes")
 
-	createConfigFromJekyll(fs, targetDir, "yaml", jekyllConfig)
+	i.createConfigFromJekyll(fs, targetDir, "yaml", jekyllConfig)
 
-	copyJekyllFilesAndFolders(jekyllRoot, filepath.Join(targetDir, "static"), jekyllPostDirs)
+	i.copyJekyllFilesAndFolders(jekyllRoot, filepath.Join(targetDir, "static"), jekyllPostDirs)
 
 	return s, nil
 }
 
-func loadJekyllConfig(fs afero.Fs, jekyllRoot string) map[string]interface{} {
+func (i *importCmd) loadJekyllConfig(fs afero.Fs, jekyllRoot string) map[string]interface{} {
 	path := filepath.Join(jekyllRoot, "_config.yml")
 
 	exists, err := helpers.Exists(path, fs)
@@ -249,16 +256,16 @@ func loadJekyllConfig(fs afero.Fs, jekyllRoot string) map[string]interface{} {
 		return nil
 	}
 
-	c, err := parser.HandleYAMLMetaData(b)
+	c, err := metadecoders.UnmarshalToMap(b, metadecoders.YAML)
 
 	if err != nil {
 		return nil
 	}
 
-	return c.(map[string]interface{})
+	return c
 }
 
-func createConfigFromJekyll(fs afero.Fs, inpath string, kind string, jekyllConfig map[string]interface{}) (err error) {
+func (i *importCmd) createConfigFromJekyll(fs afero.Fs, inpath string, kind metadecoders.Format, jekyllConfig map[string]interface{}) (err error) {
 	title := "My New Hugo Site"
 	baseURL := "http://example.org/"
 
@@ -284,15 +291,14 @@ func createConfigFromJekyll(fs afero.Fs, inpath string, kind string, jekyllConfi
 		"languageCode":       "en-us",
 		"disablePathToLower": true,
 	}
-	kind = parser.FormatSanitize(kind)
 
 	var buf bytes.Buffer
-	err = parser.InterfaceToConfig(in, parser.FormatToLeadRune(kind), &buf)
+	err = parser.InterfaceToConfig(in, kind, &buf)
 	if err != nil {
 		return err
 	}
 
-	return helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), &buf, fs)
+	return helpers.WriteToDisk(filepath.Join(inpath, "config."+string(kind)), &buf, fs)
 }
 
 func copyFile(source string, dest string) error {
@@ -353,7 +359,7 @@ func copyDir(source string, dest string) error {
 	return nil
 }
 
-func copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyllPostDirs map[string]bool) (err error) {
+func (i *importCmd) copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyllPostDirs map[string]bool) (err error) {
 	fi, err := os.Stat(jekyllRoot)
 	if err != nil {
 		return err
@@ -441,45 +447,29 @@ func convertJekyllPost(s *hugolib.Site, path, relPath, targetDir string, draft b
 		return err
 	}
 
-	psr, err := parser.ReadFrom(bytes.NewReader(contentBytes))
+	pf, err := parseContentFile(bytes.NewReader(contentBytes))
 	if err != nil {
 		jww.ERROR.Println("Parse file error:", path)
 		return err
 	}
 
-	metadata, err := psr.Metadata()
-	if err != nil {
-		jww.ERROR.Println("Processing file error:", path)
-		return err
-	}
-
-	newmetadata, err := convertJekyllMetaData(metadata, postName, postDate, draft)
+	newmetadata, err := convertJekyllMetaData(pf.frontMatter, postName, postDate, draft)
 	if err != nil {
 		jww.ERROR.Println("Convert metadata error:", path)
 		return err
 	}
 
-	jww.TRACE.Println(newmetadata)
-	content := convertJekyllContent(newmetadata, string(psr.Content()))
+	content := convertJekyllContent(newmetadata, string(pf.content))
 
-	page, err := s.NewPage(filename)
-	if err != nil {
-		jww.ERROR.Println("New page error", filename)
-		return err
+	fs := hugofs.Os
+	if err := helpers.WriteToDisk(targetFile, strings.NewReader(content), fs); err != nil {
+		return fmt.Errorf("Failed to save file %q:", filename)
 	}
 
-	page.SetSourceContent([]byte(content))
-	page.SetSourceMetaData(newmetadata, parser.FormatToLeadRune("yaml"))
-	page.SaveSourceAs(targetFile)
-
-	jww.TRACE.Println("Target file:", targetFile)
-
 	return nil
 }
 
 func convertJekyllMetaData(m interface{}, postName string, postDate time.Time, draft bool) (interface{}, error) {
-	url := postDate.Format("/2006/01/02/") + postName + "/"
-
 	metadata, err := cast.ToStringMapE(m)
 	if err != nil {
 		return nil, err
@@ -497,7 +487,7 @@ func convertJekyllMetaData(m interface{}, postName string, postDate time.Time, d
 			delete(metadata, key)
 		case "permalink":
 			if str, ok := value.(string); ok {
-				url = str
+				metadata["url"] = str
 			}
 			delete(metadata, key)
 		case "category":
@@ -526,7 +516,6 @@ func convertJekyllMetaData(m interface{}, postName string, postDate time.Time, d
 
 	}
 
-	metadata["url"] = url
 	metadata["date"] = postDate.Format(time.RFC3339)
 
 	return metadata, nil

commands/import_jekyll_test.go

@@ -51,9 +51,9 @@ func TestConvertJekyllMetadata(t *testing.T) {
 		expect   string
 	}{
 		{map[interface{}]interface{}{}, "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false,
-			`{"date":"2015-10-01T00:00:00Z","url":"/2015/10/01/testPost/"}`},
+			`{"date":"2015-10-01T00:00:00Z"}`},
 		{map[interface{}]interface{}{}, "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), true,
-			`{"date":"2015-10-01T00:00:00Z","draft":true,"url":"/2015/10/01/testPost/"}`},
+			`{"date":"2015-10-01T00:00:00Z","draft":true}`},
 		{map[interface{}]interface{}{"Permalink": "/permalink.html", "layout": "post"},
 			"testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false,
 			`{"date":"2015-10-01T00:00:00Z","url":"/permalink.html"}`},
@@ -62,13 +62,13 @@ func TestConvertJekyllMetadata(t *testing.T) {
 			`{"date":"2015-10-01T00:00:00Z","url":"/permalink.html"}`},
 		{map[interface{}]interface{}{"category": nil, "permalink": 123},
 			"testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false,
-			`{"date":"2015-10-01T00:00:00Z","url":"/2015/10/01/testPost/"}`},
+			`{"date":"2015-10-01T00:00:00Z"}`},
 		{map[interface{}]interface{}{"Excerpt_Separator": "sep"},
 			"testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false,
-			`{"date":"2015-10-01T00:00:00Z","excerpt_separator":"sep","url":"/2015/10/01/testPost/"}`},
+			`{"date":"2015-10-01T00:00:00Z","excerpt_separator":"sep"}`},
 		{map[interface{}]interface{}{"category": "book", "layout": "post", "Others": "Goods", "Date": "2015-10-01 12:13:11"},
 			"testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false,
-			`{"Others":"Goods","categories":["book"],"date":"2015-10-01T12:13:11Z","url":"/2015/10/01/testPost/"}`},
+			`{"Others":"Goods","categories":["book"],"date":"2015-10-01T12:13:11Z"}`},
 	}
 
 	for _, data := range testDataList {

commands/limit_darwin.go

@@ -1,17 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -33,39 +20,45 @@ import (
 	jww "github.com/spf13/jwalterweatherman"
 )
 
-func init() {
-	checkCmd.AddCommand(limit)
+var _ cmder = (*limitCmd)(nil)
+
+type limitCmd struct {
+	*baseCmd
 }
 
-var limit = &cobra.Command{
-	Use:   "ulimit",
-	Short: "Check system ulimit settings",
-	Long: `Hugo will inspect the current ulimit settings on the system.
+func newLimitCmd() *limitCmd {
+	ccmd := &cobra.Command{
+		Use:   "ulimit",
+		Short: "Check system ulimit settings",
+		Long: `Hugo will inspect the current ulimit settings on the system.
 This is primarily to ensure that Hugo can watch enough files on some OSs`,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		var rLimit syscall.Rlimit
-		err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
-		if err != nil {
-			return newSystemError("Error Getting Rlimit ", err)
-		}
+		RunE: func(cmd *cobra.Command, args []string) error {
+			var rLimit syscall.Rlimit
+			err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+			if err != nil {
+				return newSystemError("Error Getting Rlimit ", err)
+			}
 
-		jww.FEEDBACK.Println("Current rLimit:", rLimit)
+			jww.FEEDBACK.Println("Current rLimit:", rLimit)
 
-		jww.FEEDBACK.Println("Attempting to increase limit")
-		rLimit.Max = 999999
-		rLimit.Cur = 999999
-		err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
-		if err != nil {
-			return newSystemError("Error Setting rLimit ", err)
-		}
-		err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
-		if err != nil {
-			return newSystemError("Error Getting rLimit ", err)
-		}
-		jww.FEEDBACK.Println("rLimit after change:", rLimit)
+			jww.FEEDBACK.Println("Attempting to increase limit")
+			rLimit.Max = 999999
+			rLimit.Cur = 999999
+			err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+			if err != nil {
+				return newSystemError("Error Setting rLimit ", err)
+			}
+			err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+			if err != nil {
+				return newSystemError("Error Getting rLimit ", err)
+			}
+			jww.FEEDBACK.Println("rLimit after change:", rLimit)
 
-		return nil
-	},
+			return nil
+		},
+	}
+
+	return &limitCmd{baseCmd: newBaseCmd(ccmd)}
 }
 
 func tweakLimit() {

commands/limit_others.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,6 @@
 // limitations under the License.
 
 // +build !darwin
-// Copyright 2015 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 package commands
 

commands/list.go

@@ -21,129 +21,137 @@ import (
 	jww "github.com/spf13/jwalterweatherman"
 )
 
-func init() {
-	listCmd.AddCommand(listDraftsCmd)
-	listCmd.AddCommand(listFutureCmd)
-	listCmd.AddCommand(listExpiredCmd)
-	listCmd.PersistentFlags().StringVarP(&source, "source", "s", "", "filesystem path to read files relative from")
-	listCmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{})
+var _ cmder = (*listCmd)(nil)
+
+type listCmd struct {
+	hugoBuilderCommon
+	*baseCmd
 }
 
-var listCmd = &cobra.Command{
-	Use:   "list",
-	Short: "Listing out various types of content",
-	Long: `Listing out various types of content.
+func newListCmd() *listCmd {
+	cc := &listCmd{}
+
+	cc.baseCmd = newBaseCmd(&cobra.Command{
+		Use:   "list",
+		Short: "Listing out various types of content",
+		Long: `Listing out various types of content.
 
 List requires a subcommand, e.g. ` + "`hugo list drafts`.",
-	RunE: nil,
-}
+		RunE: nil,
+	})
 
-var listDraftsCmd = &cobra.Command{
-	Use:   "drafts",
-	Short: "List all drafts",
-	Long:  `List all of the drafts in your content directory.`,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		cfgInit := func(c *commandeer) error {
-			c.Set("buildDrafts", true)
-			return nil
-		}
-		c, err := InitializeConfig(false, cfgInit)
-		if err != nil {
-			return err
-		}
+	cc.cmd.AddCommand(
+		&cobra.Command{
+			Use:   "drafts",
+			Short: "List all drafts",
+			Long:  `List all of the drafts in your content directory.`,
+			RunE: func(cmd *cobra.Command, args []string) error {
+				cfgInit := func(c *commandeer) error {
+					c.Set("buildDrafts", true)
+					return nil
+				}
+				c, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, cfgInit)
+				if err != nil {
+					return err
+				}
 
-		sites, err := hugolib.NewHugoSites(*c.DepsCfg)
+				sites, err := hugolib.NewHugoSites(*c.DepsCfg)
 
-		if err != nil {
-			return newSystemError("Error creating sites", err)
-		}
+				if err != nil {
+					return newSystemError("Error creating sites", err)
+				}
 
-		if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
-			return newSystemError("Error Processing Source Content", err)
-		}
+				if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
+					return newSystemError("Error Processing Source Content", err)
+				}
 
-		for _, p := range sites.Pages() {
-			if p.IsDraft() {
-				jww.FEEDBACK.Println(filepath.Join(p.File.Dir(), p.File.LogicalName()))
-			}
+				for _, p := range sites.Pages() {
+					if p.IsDraft() {
+						jww.FEEDBACK.Println(filepath.Join(p.File.Dir(), p.File.LogicalName()))
+					}
 
-		}
+				}
 
-		return nil
+				return nil
 
-	},
-}
-
-var listFutureCmd = &cobra.Command{
-	Use:   "future",
-	Short: "List all posts dated in the future",
-	Long: `List all of the posts in your content directory which will be
+			},
+		},
+		&cobra.Command{
+			Use:   "future",
+			Short: "List all posts dated in the future",
+			Long: `List all of the posts in your content directory which will be
 posted in the future.`,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		cfgInit := func(c *commandeer) error {
-			c.Set("buildFuture", true)
-			return nil
-		}
-		c, err := InitializeConfig(false, cfgInit)
-		if err != nil {
-			return err
-		}
+			RunE: func(cmd *cobra.Command, args []string) error {
+				cfgInit := func(c *commandeer) error {
+					c.Set("buildFuture", true)
+					return nil
+				}
+				c, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, cfgInit)
+				if err != nil {
+					return err
+				}
 
-		sites, err := hugolib.NewHugoSites(*c.DepsCfg)
+				sites, err := hugolib.NewHugoSites(*c.DepsCfg)
 
-		if err != nil {
-			return newSystemError("Error creating sites", err)
-		}
+				if err != nil {
+					return newSystemError("Error creating sites", err)
+				}
 
-		if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
-			return newSystemError("Error Processing Source Content", err)
-		}
+				if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
+					return newSystemError("Error Processing Source Content", err)
+				}
 
-		for _, p := range sites.Pages() {
-			if p.IsFuture() {
-				jww.FEEDBACK.Println(filepath.Join(p.File.Dir(), p.File.LogicalName()))
-			}
+				for _, p := range sites.Pages() {
+					if p.IsFuture() {
+						jww.FEEDBACK.Println(filepath.Join(p.File.Dir(), p.File.LogicalName()))
+					}
 
-		}
+				}
 
-		return nil
+				return nil
 
-	},
-}
-
-var listExpiredCmd = &cobra.Command{
-	Use:   "expired",
-	Short: "List all posts already expired",
-	Long: `List all of the posts in your content directory which has already
+			},
+		},
+		&cobra.Command{
+			Use:   "expired",
+			Short: "List all posts already expired",
+			Long: `List all of the posts in your content directory which has already
 expired.`,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		cfgInit := func(c *commandeer) error {
-			c.Set("buildExpired", true)
-			return nil
-		}
-		c, err := InitializeConfig(false, cfgInit)
-		if err != nil {
-			return err
-		}
+			RunE: func(cmd *cobra.Command, args []string) error {
+				cfgInit := func(c *commandeer) error {
+					c.Set("buildExpired", true)
+					return nil
+				}
+				c, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, cfgInit)
+				if err != nil {
+					return err
+				}
 
-		sites, err := hugolib.NewHugoSites(*c.DepsCfg)
+				sites, err := hugolib.NewHugoSites(*c.DepsCfg)
 
-		if err != nil {
-			return newSystemError("Error creating sites", err)
-		}
+				if err != nil {
+					return newSystemError("Error creating sites", err)
+				}
 
-		if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
-			return newSystemError("Error Processing Source Content", err)
-		}
+				if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
+					return newSystemError("Error Processing Source Content", err)
+				}
 
-		for _, p := range sites.Pages() {
-			if p.IsExpired() {
-				jww.FEEDBACK.Println(filepath.Join(p.File.Dir(), p.File.LogicalName()))
-			}
+				for _, p := range sites.Pages() {
+					if p.IsExpired() {
+						jww.FEEDBACK.Println(filepath.Join(p.File.Dir(), p.File.LogicalName()))
+					}
 
-		}
+				}
 
-		return nil
+				return nil
 
-	},
+			},
+		},
+	)
+
+	cc.cmd.PersistentFlags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")
+	cc.cmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{})
+
+	return cc
 }

commands/new.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,85 +15,65 @@ package commands
 
 import (
 	"bytes"
-	"errors"
-	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/gohugoio/hugo/create"
 	"github.com/gohugoio/hugo/helpers"
-	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/hugolib"
-	"github.com/gohugoio/hugo/parser"
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 	jww "github.com/spf13/jwalterweatherman"
-	"github.com/spf13/viper"
 )
 
-var (
-	configFormat  string
+var _ cmder = (*newCmd)(nil)
+
+type newCmd struct {
 	contentEditor string
 	contentType   string
-)
-
-func init() {
-	newSiteCmd.Flags().StringVarP(&configFormat, "format", "f", "toml", "config & frontmatter format")
-	newSiteCmd.Flags().Bool("force", false, "init inside non-empty directory")
-	newCmd.Flags().StringVarP(&contentType, "kind", "k", "", "content type to create")
-	newCmd.PersistentFlags().StringVarP(&source, "source", "s", "", "filesystem path to read files relative from")
-	newCmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{})
-	newCmd.Flags().StringVar(&contentEditor, "editor", "", "edit new content with this editor, if provided")
-
-	newCmd.AddCommand(newSiteCmd)
-	newCmd.AddCommand(newThemeCmd)
 
+	*baseBuilderCmd
 }
 
-var newCmd = &cobra.Command{
-	Use:   "new [path]",
-	Short: "Create new content for your site",
-	Long: `Create a new content file and automatically set the date and title.
+func (b *commandsBuilder) newNewCmd() *newCmd {
+	cmd := &cobra.Command{
+		Use:   "new [path]",
+		Short: "Create new content for your site",
+		Long: `Create a new content file and automatically set the date and title.
 It will guess which kind of file to create based on the path provided.
 
 You can also specify the kind with ` + "`-k KIND`" + `.
 
-If archetypes are provided in your theme or site, they will be used.`,
+If archetypes are provided in your theme or site, they will be used.
 
-	RunE: NewContent,
+Ensure you run this within the root directory of your site.`,
+	}
+
+	cc := &newCmd{baseBuilderCmd: b.newBuilderCmd(cmd)}
+
+	cmd.Flags().StringVarP(&cc.contentType, "kind", "k", "", "content type to create")
+	cmd.PersistentFlags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from")
+	cmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{})
+	cmd.Flags().StringVar(&cc.contentEditor, "editor", "", "edit new content with this editor, if provided")
+
+	cmd.AddCommand(newNewSiteCmd().getCommand())
+	cmd.AddCommand(newNewThemeCmd().getCommand())
+
+	cmd.RunE = cc.newContent
+
+	return cc
 }
 
-var newSiteCmd = &cobra.Command{
-	Use:   "site [path]",
-	Short: "Create a new site (skeleton)",
-	Long: `Create a new site in the provided directory.
-The new site will have the correct structure, but no content or theme yet.
-Use ` + "`hugo new [contentPath]`" + ` to create new content.`,
-	RunE: NewSite,
-}
-
-var newThemeCmd = &cobra.Command{
-	Use:   "theme [name]",
-	Short: "Create a new theme",
-	Long: `Create a new theme (skeleton) called [name] in the current directory.
-New theme is a skeleton. Please add content to the touched files. Add your
-name to the copyright line in the license and adjust the theme.toml file
-as you see fit.`,
-	RunE: NewTheme,
-}
-
-// NewContent adds new content to a Hugo site.
-func NewContent(cmd *cobra.Command, args []string) error {
+func (n *newCmd) newContent(cmd *cobra.Command, args []string) error {
 	cfgInit := func(c *commandeer) error {
 		if cmd.Flags().Changed("editor") {
-			c.Set("newContentEditor", contentEditor)
+			c.Set("newContentEditor", n.contentEditor)
 		}
 		return nil
 	}
 
-	c, err := InitializeConfig(false, cfgInit)
+	c, err := initializeConfig(true, false, &n.hugoBuilderCommon, n, cfgInit)
 
 	if err != nil {
 		return err
@@ -107,211 +87,13 @@ func NewContent(cmd *cobra.Command, args []string) error {
 
 	var kind string
 
-	createPath, kind = newContentPathSection(createPath)
+	createPath, kind = newContentPathSection(c.hugo, createPath)
 
-	if contentType != "" {
-		kind = contentType
+	if n.contentType != "" {
+		kind = n.contentType
 	}
 
-	cfg := c.DepsCfg
-
-	ps, err := helpers.NewPathSpec(cfg.Fs, cfg.Cfg)
-	if err != nil {
-		return err
-	}
-
-	// If a site isn't in use in the archetype template, we can skip the build.
-	siteFactory := func(filename string, siteUsed bool) (*hugolib.Site, error) {
-		if !siteUsed {
-			return hugolib.NewSite(*cfg)
-		}
-		var s *hugolib.Site
-		if err := c.initSites(); err != nil {
-			return nil, err
-		}
-
-		if err := Hugo.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
-			return nil, err
-		}
-
-		s = Hugo.Sites[0]
-
-		if len(Hugo.Sites) > 1 {
-			// Find the best match.
-			for _, ss := range Hugo.Sites {
-				if strings.Contains(createPath, "."+ss.Language.Lang) {
-					s = ss
-					break
-				}
-			}
-		}
-		return s, nil
-	}
-
-	return create.NewContent(ps, siteFactory, kind, createPath)
-}
-
-func doNewSite(fs *hugofs.Fs, basepath string, force bool) error {
-	archeTypePath := filepath.Join(basepath, "archetypes")
-	dirs := []string{
-		filepath.Join(basepath, "layouts"),
-		filepath.Join(basepath, "content"),
-		archeTypePath,
-		filepath.Join(basepath, "static"),
-		filepath.Join(basepath, "data"),
-		filepath.Join(basepath, "themes"),
-	}
-
-	if exists, _ := helpers.Exists(basepath, fs.Source); exists {
-		if isDir, _ := helpers.IsDir(basepath, fs.Source); !isDir {
-			return errors.New(basepath + " already exists but not a directory")
-		}
-
-		isEmpty, _ := helpers.IsEmpty(basepath, fs.Source)
-
-		switch {
-		case !isEmpty && !force:
-			return errors.New(basepath + " already exists and is not empty")
-
-		case !isEmpty && force:
-			all := append(dirs, filepath.Join(basepath, "config."+configFormat))
-			for _, path := range all {
-				if exists, _ := helpers.Exists(path, fs.Source); exists {
-					return errors.New(path + " already exists")
-				}
-			}
-		}
-	}
-
-	for _, dir := range dirs {
-		if err := fs.Source.MkdirAll(dir, 0777); err != nil {
-			return fmt.Errorf("Failed to create dir: %s", err)
-		}
-	}
-
-	createConfig(fs, basepath, configFormat)
-
-	// Create a defaul archetype file.
-	helpers.SafeWriteToDisk(filepath.Join(archeTypePath, "default.md"),
-		strings.NewReader(create.ArchetypeTemplateTemplate), fs.Source)
-
-	jww.FEEDBACK.Printf("Congratulations! Your new Hugo site is created in %s.\n\n", basepath)
-	jww.FEEDBACK.Println(nextStepsText())
-
-	return nil
-}
-
-func nextStepsText() string {
-	var nextStepsText bytes.Buffer
-
-	nextStepsText.WriteString(`Just a few more steps and you're ready to go:
-
-1. Download a theme into the same-named folder.
-   Choose a theme from https://themes.gohugo.io/, or
-   create your own with the "hugo new theme <THEMENAME>" command.
-2. Perhaps you want to add some content. You can add single files
-   with "hugo new `)
-
-	nextStepsText.WriteString(filepath.Join("<SECTIONNAME>", "<FILENAME>.<FORMAT>"))
-
-	nextStepsText.WriteString(`".
-3. Start the built-in live server via "hugo server".
-
-Visit https://gohugo.io/ for quickstart guide and full documentation.`)
-
-	return nextStepsText.String()
-}
-
-// NewSite creates a new Hugo site and initializes a structured Hugo directory.
-func NewSite(cmd *cobra.Command, args []string) error {
-	if len(args) < 1 {
-		return newUserError("path needs to be provided")
-	}
-
-	createpath, err := filepath.Abs(filepath.Clean(args[0]))
-	if err != nil {
-		return newUserError(err)
-	}
-
-	forceNew, _ := cmd.Flags().GetBool("force")
-
-	return doNewSite(hugofs.NewDefault(viper.New()), createpath, forceNew)
-}
-
-// NewTheme creates a new Hugo theme.
-func NewTheme(cmd *cobra.Command, args []string) error {
-	c, err := InitializeConfig(false, nil)
-
-	if err != nil {
-		return err
-	}
-
-	if len(args) < 1 {
-		return newUserError("theme name needs to be provided")
-	}
-
-	createpath := c.PathSpec().AbsPathify(filepath.Join(c.Cfg.GetString("themesDir"), args[0]))
-	jww.INFO.Println("creating theme at", createpath)
-
-	cfg := c.DepsCfg
-
-	if x, _ := helpers.Exists(createpath, cfg.Fs.Source); x {
-		return newUserError(createpath, "already exists")
-	}
-
-	mkdir(createpath, "layouts", "_default")
-	mkdir(createpath, "layouts", "partials")
-
-	touchFile(cfg.Fs.Source, createpath, "layouts", "index.html")
-	touchFile(cfg.Fs.Source, createpath, "layouts", "404.html")
-	touchFile(cfg.Fs.Source, createpath, "layouts", "_default", "list.html")
-	touchFile(cfg.Fs.Source, createpath, "layouts", "_default", "single.html")
-
-	touchFile(cfg.Fs.Source, createpath, "layouts", "partials", "header.html")
-	touchFile(cfg.Fs.Source, createpath, "layouts", "partials", "footer.html")
-
-	mkdir(createpath, "archetypes")
-
-	archDefault := []byte("+++\n+++\n")
-
-	err = helpers.WriteToDisk(filepath.Join(createpath, "archetypes", "default.md"), bytes.NewReader(archDefault), cfg.Fs.Source)
-	if err != nil {
-		return err
-	}
-
-	mkdir(createpath, "static", "js")
-	mkdir(createpath, "static", "css")
-
-	by := []byte(`The MIT License (MIT)
-
-Copyright (c) ` + time.Now().Format("2006") + ` YOUR_NAME_HERE
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-`)
-
-	err = helpers.WriteToDisk(filepath.Join(createpath, "LICENSE.md"), bytes.NewReader(by), cfg.Fs.Source)
-	if err != nil {
-		return err
-	}
-
-	createThemeMD(cfg.Fs, createpath)
-
-	return nil
+	return create.NewContent(c.hugo, kind, createPath)
 }
 
 func mkdir(x ...string) {
@@ -332,43 +114,17 @@ func touchFile(fs afero.Fs, x ...string) {
 	}
 }
 
-func createThemeMD(fs *hugofs.Fs, inpath string) (err error) {
-
-	by := []byte(`# theme.toml template for a Hugo theme
-# See https://github.com/gohugoio/hugoThemes#themetoml for an example
-
-name = "` + strings.Title(helpers.MakeTitle(filepath.Base(inpath))) + `"
-license = "MIT"
-licenselink = "https://github.com/yourname/yourtheme/blob/master/LICENSE.md"
-description = ""
-homepage = "http://example.com/"
-tags = []
-features = []
-min_version = "0.34"
-
-[author]
-  name = ""
-  homepage = ""
-
-# If porting an existing theme
-[original]
-  name = ""
-  homepage = ""
-  repo = ""
-`)
-
-	err = helpers.WriteToDisk(filepath.Join(inpath, "theme.toml"), bytes.NewReader(by), fs.Source)
-	if err != nil {
-		return
-	}
-
-	return nil
-}
-
-func newContentPathSection(path string) (string, string) {
+func newContentPathSection(h *hugolib.HugoSites, path string) (string, string) {
 	// Forward slashes is used in all examples. Convert if needed.
 	// Issue #1133
 	createpath := filepath.FromSlash(path)
+
+	if h != nil {
+		for _, s := range h.Sites {
+			createpath = strings.TrimPrefix(createpath, s.PathSpec.ContentDir)
+		}
+	}
+
 	var section string
 	// assume the first directory is the section (kind)
 	if strings.Contains(createpath[1:], helpers.FilePathSeparator) {
@@ -381,20 +137,3 @@ func newContentPathSection(path string) (string, string) {
 
 	return createpath, section
 }
-
-func createConfig(fs *hugofs.Fs, inpath string, kind string) (err error) {
-	in := map[string]string{
-		"baseURL":      "http://example.org/",
-		"title":        "My New Hugo Site",
-		"languageCode": "en-us",
-	}
-	kind = parser.FormatSanitize(kind)
-
-	var buf bytes.Buffer
-	err = parser.InterfaceToConfig(in, parser.FormatToLeadRune(kind), &buf)
-	if err != nil {
-		return err
-	}
-
-	return helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), &buf, fs.Source)
-}

commands/new_content_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -25,7 +25,7 @@ import (
 
 // Issue #1133
 func TestNewContentPathSectionWithForwardSlashes(t *testing.T) {
-	p, s := newContentPathSection("/post/new.md")
+	p, s := newContentPathSection(nil, "/post/new.md")
 	assert.Equal(t, filepath.FromSlash("/post/new.md"), p)
 	assert.Equal(t, "post", s)
 }
@@ -48,10 +48,11 @@ func checkNewSiteInited(fs *hugofs.Fs, basepath string, t *testing.T) {
 }
 
 func TestDoNewSite(t *testing.T) {
+	n := newNewSiteCmd()
 	basepath := filepath.Join("base", "blog")
 	_, fs := newTestCfg()
 
-	require.NoError(t, doNewSite(fs, basepath, false))
+	require.NoError(t, n.doNewSite(fs, basepath, false))
 
 	checkNewSiteInited(fs, basepath, t)
 }
@@ -59,31 +60,34 @@ func TestDoNewSite(t *testing.T) {
 func TestDoNewSite_noerror_base_exists_but_empty(t *testing.T) {
 	basepath := filepath.Join("base", "blog")
 	_, fs := newTestCfg()
+	n := newNewSiteCmd()
 
 	require.NoError(t, fs.Source.MkdirAll(basepath, 777))
 
-	require.NoError(t, doNewSite(fs, basepath, false))
+	require.NoError(t, n.doNewSite(fs, basepath, false))
 }
 
 func TestDoNewSite_error_base_exists(t *testing.T) {
 	basepath := filepath.Join("base", "blog")
 	_, fs := newTestCfg()
+	n := newNewSiteCmd()
 
 	require.NoError(t, fs.Source.MkdirAll(basepath, 777))
 	_, err := fs.Source.Create(filepath.Join(basepath, "foo"))
 	require.NoError(t, err)
 	// Since the directory already exists and isn't empty, expect an error
-	require.Error(t, doNewSite(fs, basepath, false))
+	require.Error(t, n.doNewSite(fs, basepath, false))
 
 }
 
 func TestDoNewSite_force_empty_dir(t *testing.T) {
 	basepath := filepath.Join("base", "blog")
 	_, fs := newTestCfg()
+	n := newNewSiteCmd()
 
 	require.NoError(t, fs.Source.MkdirAll(basepath, 777))
 
-	require.NoError(t, doNewSite(fs, basepath, true))
+	require.NoError(t, n.doNewSite(fs, basepath, true))
 
 	checkNewSiteInited(fs, basepath, t)
 }
@@ -91,23 +95,25 @@ func TestDoNewSite_force_empty_dir(t *testing.T) {
 func TestDoNewSite_error_force_dir_inside_exists(t *testing.T) {
 	basepath := filepath.Join("base", "blog")
 	_, fs := newTestCfg()
+	n := newNewSiteCmd()
 
 	contentPath := filepath.Join(basepath, "content")
 
 	require.NoError(t, fs.Source.MkdirAll(contentPath, 777))
-	require.Error(t, doNewSite(fs, basepath, true))
+	require.Error(t, n.doNewSite(fs, basepath, true))
 }
 
 func TestDoNewSite_error_force_config_inside_exists(t *testing.T) {
 	basepath := filepath.Join("base", "blog")
 	_, fs := newTestCfg()
+	n := newNewSiteCmd()
 
 	configPath := filepath.Join(basepath, "config.toml")
 	require.NoError(t, fs.Source.MkdirAll(basepath, 777))
 	_, err := fs.Source.Create(configPath)
 	require.NoError(t, err)
 
-	require.Error(t, doNewSite(fs, basepath, true))
+	require.Error(t, n.doNewSite(fs, basepath, true))
 }
 
 func newTestCfg() (*viper.Viper, *hugofs.Fs) {

commands/new_site.go

@@ -0,0 +1,165 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package commands
+
+import (
+	"bytes"
+	"errors"
+	"path/filepath"
+	"strings"
+
+	"github.com/gohugoio/hugo/parser/metadecoders"
+
+	_errors "github.com/pkg/errors"
+
+	"github.com/gohugoio/hugo/create"
+	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/hugofs"
+	"github.com/gohugoio/hugo/parser"
+	"github.com/spf13/cobra"
+	jww "github.com/spf13/jwalterweatherman"
+	"github.com/spf13/viper"
+)
+
+var _ cmder = (*newSiteCmd)(nil)
+
+type newSiteCmd struct {
+	configFormat string
+
+	*baseCmd
+}
+
+func newNewSiteCmd() *newSiteCmd {
+	ccmd := &newSiteCmd{}
+
+	cmd := &cobra.Command{
+		Use:   "site [path]",
+		Short: "Create a new site (skeleton)",
+		Long: `Create a new site in the provided directory.
+The new site will have the correct structure, but no content or theme yet.
+Use ` + "`hugo new [contentPath]`" + ` to create new content.`,
+		RunE: ccmd.newSite,
+	}
+
+	cmd.Flags().StringVarP(&ccmd.configFormat, "format", "f", "toml", "config & frontmatter format")
+	cmd.Flags().Bool("force", false, "init inside non-empty directory")
+
+	ccmd.baseCmd = newBaseCmd(cmd)
+
+	return ccmd
+
+}
+
+func (n *newSiteCmd) doNewSite(fs *hugofs.Fs, basepath string, force bool) error {
+	archeTypePath := filepath.Join(basepath, "archetypes")
+	dirs := []string{
+		filepath.Join(basepath, "layouts"),
+		filepath.Join(basepath, "content"),
+		archeTypePath,
+		filepath.Join(basepath, "static"),
+		filepath.Join(basepath, "data"),
+		filepath.Join(basepath, "themes"),
+	}
+
+	if exists, _ := helpers.Exists(basepath, fs.Source); exists {
+		if isDir, _ := helpers.IsDir(basepath, fs.Source); !isDir {
+			return errors.New(basepath + " already exists but not a directory")
+		}
+
+		isEmpty, _ := helpers.IsEmpty(basepath, fs.Source)
+
+		switch {
+		case !isEmpty && !force:
+			return errors.New(basepath + " already exists and is not empty")
+
+		case !isEmpty && force:
+			all := append(dirs, filepath.Join(basepath, "config."+n.configFormat))
+			for _, path := range all {
+				if exists, _ := helpers.Exists(path, fs.Source); exists {
+					return errors.New(path + " already exists")
+				}
+			}
+		}
+	}
+
+	for _, dir := range dirs {
+		if err := fs.Source.MkdirAll(dir, 0777); err != nil {
+			return _errors.Wrap(err, "Failed to create dir")
+		}
+	}
+
+	createConfig(fs, basepath, n.configFormat)
+
+	// Create a default archetype file.
+	helpers.SafeWriteToDisk(filepath.Join(archeTypePath, "default.md"),
+		strings.NewReader(create.ArchetypeTemplateTemplate), fs.Source)
+
+	jww.FEEDBACK.Printf("Congratulations! Your new Hugo site is created in %s.\n\n", basepath)
+	jww.FEEDBACK.Println(nextStepsText())
+
+	return nil
+}
+
+// newSite creates a new Hugo site and initializes a structured Hugo directory.
+func (n *newSiteCmd) newSite(cmd *cobra.Command, args []string) error {
+	if len(args) < 1 {
+		return newUserError("path needs to be provided")
+	}
+
+	createpath, err := filepath.Abs(filepath.Clean(args[0]))
+	if err != nil {
+		return newUserError(err)
+	}
+
+	forceNew, _ := cmd.Flags().GetBool("force")
+
+	return n.doNewSite(hugofs.NewDefault(viper.New()), createpath, forceNew)
+}
+
+func createConfig(fs *hugofs.Fs, inpath string, kind string) (err error) {
+	in := map[string]string{
+		"baseURL":      "http://example.org/",
+		"title":        "My New Hugo Site",
+		"languageCode": "en-us",
+	}
+
+	var buf bytes.Buffer
+	err = parser.InterfaceToConfig(in, metadecoders.FormatFromString(kind), &buf)
+	if err != nil {
+		return err
+	}
+
+	return helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), &buf, fs.Source)
+}
+
+func nextStepsText() string {
+	var nextStepsText bytes.Buffer
+
+	nextStepsText.WriteString(`Just a few more steps and you're ready to go:
+
+1. Download a theme into the same-named folder.
+   Choose a theme from https://themes.gohugo.io/, or
+   create your own with the "hugo new theme <THEMENAME>" command.
+2. Perhaps you want to add some content. You can add single files
+   with "hugo new `)
+
+	nextStepsText.WriteString(filepath.Join("<SECTIONNAME>", "<FILENAME>.<FORMAT>"))
+
+	nextStepsText.WriteString(`".
+3. Start the built-in live server via "hugo server".
+
+Visit https://gohugo.io/ for quickstart guide and full documentation.`)
+
+	return nextStepsText.String()
+}

commands/new_theme.go

@@ -0,0 +1,179 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package commands
+
+import (
+	"bytes"
+	"errors"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/hugofs"
+	"github.com/spf13/cobra"
+	jww "github.com/spf13/jwalterweatherman"
+)
+
+var _ cmder = (*newThemeCmd)(nil)
+
+type newThemeCmd struct {
+	*baseCmd
+	hugoBuilderCommon
+}
+
+func newNewThemeCmd() *newThemeCmd {
+	ccmd := &newThemeCmd{baseCmd: newBaseCmd(nil)}
+
+	cmd := &cobra.Command{
+		Use:   "theme [name]",
+		Short: "Create a new theme",
+		Long: `Create a new theme (skeleton) called [name] in the current directory.
+New theme is a skeleton. Please add content to the touched files. Add your
+name to the copyright line in the license and adjust the theme.toml file
+as you see fit.`,
+		RunE: ccmd.newTheme,
+	}
+
+	ccmd.cmd = cmd
+
+	return ccmd
+}
+
+// newTheme creates a new Hugo theme template
+func (n *newThemeCmd) newTheme(cmd *cobra.Command, args []string) error {
+	c, err := initializeConfig(false, false, &n.hugoBuilderCommon, n, nil)
+
+	if err != nil {
+		return err
+	}
+
+	if len(args) < 1 {
+		return newUserError("theme name needs to be provided")
+	}
+
+	createpath := c.hugo.PathSpec.AbsPathify(filepath.Join(c.Cfg.GetString("themesDir"), args[0]))
+	jww.FEEDBACK.Println("Creating theme at", createpath)
+
+	cfg := c.DepsCfg
+
+	if x, _ := helpers.Exists(createpath, cfg.Fs.Source); x {
+		return errors.New(createpath + " already exists")
+	}
+
+	mkdir(createpath, "layouts", "_default")
+	mkdir(createpath, "layouts", "partials")
+
+	touchFile(cfg.Fs.Source, createpath, "layouts", "index.html")
+	touchFile(cfg.Fs.Source, createpath, "layouts", "404.html")
+	touchFile(cfg.Fs.Source, createpath, "layouts", "_default", "list.html")
+	touchFile(cfg.Fs.Source, createpath, "layouts", "_default", "single.html")
+
+	baseofDefault := []byte(`<!DOCTYPE html>
+<html>
+    {{- partial "head.html" . -}}
+    <body>
+        {{- partial "header.html" . -}}
+        <div id="content">
+        {{- block "main" . }}{{- end }}
+        </div>
+        {{- partial "footer.html" . -}}
+    </body>
+</html>
+`)
+	err = helpers.WriteToDisk(filepath.Join(createpath, "layouts", "_default", "baseof.html"), bytes.NewReader(baseofDefault), cfg.Fs.Source)
+	if err != nil {
+		return err
+	}
+
+	touchFile(cfg.Fs.Source, createpath, "layouts", "partials", "head.html")
+	touchFile(cfg.Fs.Source, createpath, "layouts", "partials", "header.html")
+	touchFile(cfg.Fs.Source, createpath, "layouts", "partials", "footer.html")
+
+	mkdir(createpath, "archetypes")
+
+	archDefault := []byte("+++\n+++\n")
+
+	err = helpers.WriteToDisk(filepath.Join(createpath, "archetypes", "default.md"), bytes.NewReader(archDefault), cfg.Fs.Source)
+	if err != nil {
+		return err
+	}
+
+	mkdir(createpath, "static", "js")
+	mkdir(createpath, "static", "css")
+
+	by := []byte(`The MIT License (MIT)
+
+Copyright (c) ` + time.Now().Format("2006") + ` YOUR_NAME_HERE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+`)
+
+	err = helpers.WriteToDisk(filepath.Join(createpath, "LICENSE"), bytes.NewReader(by), cfg.Fs.Source)
+	if err != nil {
+		return err
+	}
+
+	n.createThemeMD(cfg.Fs, createpath)
+
+	return nil
+}
+
+func (n *newThemeCmd) createThemeMD(fs *hugofs.Fs, inpath string) (err error) {
+
+	by := []byte(`# theme.toml template for a Hugo theme
+# See https://github.com/gohugoio/hugoThemes#themetoml for an example
+
+name = "` + strings.Title(helpers.MakeTitle(filepath.Base(inpath))) + `"
+license = "MIT"
+licenselink = "https://github.com/yourname/yourtheme/blob/master/LICENSE"
+description = ""
+homepage = "http://example.com/"
+tags = []
+features = []
+min_version = "0.41"
+
+[author]
+  name = ""
+  homepage = ""
+
+# If porting an existing theme
+[original]
+  name = ""
+  homepage = ""
+  repo = ""
+`)
+
+	err = helpers.WriteToDisk(filepath.Join(inpath, "theme.toml"), bytes.NewReader(by), fs.Source)
+	if err != nil {
+		return
+	}
+
+	return nil
+}

commands/release.go

@@ -18,13 +18,12 @@ package commands
 import (
 	"errors"
 
+	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/releaser"
 	"github.com/spf13/cobra"
 )
 
-func init() {
-	HugoCmd.AddCommand(createReleaser().cmd)
-}
+var _ cmder = (*releaseCommandeer)(nil)
 
 type releaseCommandeer struct {
 	cmd *cobra.Command
@@ -35,7 +34,7 @@ type releaseCommandeer struct {
 	try         bool
 }
 
-func createReleaser() *releaseCommandeer {
+func createReleaser() cmder {
 	// Note: This is a command only meant for internal use and must be run
 	// via "go run -tags release main.go release" on the actual code base that is in the release.
 	r := &releaseCommandeer{
@@ -57,6 +56,14 @@ func createReleaser() *releaseCommandeer {
 	return r
 }
 
+func (c *releaseCommandeer) getCommand() *cobra.Command {
+	return c.cmd
+}
+
+func (c *releaseCommandeer) flagsToConfig(cfg config.Provider) {
+
+}
+
 func (r *releaseCommandeer) release() error {
 	if r.version == "" {
 		return errors.New("must set the --rel flag to the relevant version number")

commands/release_noop.go

@@ -0,0 +1,20 @@
+// +build !release
+
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package commands
+
+func createReleaser() cmder {
+	return &nilCommand{}
+}

commands/server.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
 package commands
 
 import (
+	"bytes"
 	"fmt"
 	"net"
 	"net/http"
@@ -21,13 +22,18 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/gohugoio/hugo/livereload"
+	"github.com/gohugoio/hugo/tpl"
 
 	"github.com/gohugoio/hugo/config"
 
@@ -37,7 +43,10 @@ import (
 	jww "github.com/spf13/jwalterweatherman"
 )
 
-var (
+type serverCmd struct {
+	// Can be used to stop the server. Useful in tests
+	stop <-chan bool
+
 	disableLiveReload bool
 	navigateToChanged bool
 	renderToDisk      bool
@@ -48,14 +57,24 @@ var (
 	serverWatch       bool
 	noHTTPCache       bool
 
-	disableFastRender bool
-)
+	disableFastRender   bool
+	disableBrowserError bool
 
-var serverCmd = &cobra.Command{
-	Use:     "server",
-	Aliases: []string{"serve"},
-	Short:   "A high performance webserver",
-	Long: `Hugo provides its own webserver which builds and serves the site.
+	*baseBuilderCmd
+}
+
+func (b *commandsBuilder) newServerCmd() *serverCmd {
+	return b.newServerCmdSignaled(nil)
+}
+
+func (b *commandsBuilder) newServerCmdSignaled(stop <-chan bool) *serverCmd {
+	cc := &serverCmd{stop: stop}
+
+	cc.baseBuilderCmd = b.newBuilderCmd(&cobra.Command{
+		Use:     "server",
+		Aliases: []string{"serve"},
+		Short:   "A high performance webserver",
+		Long: `Hugo provides its own webserver which builds and serves the site.
 While hugo server is high performance, it is a webserver with limited options.
 Many run it in production, but the standard behavior is for people to use it
 in development and use a more full featured server such as Nginx or Caddy.
@@ -67,7 +86,25 @@ By default hugo will also watch your files for any changes you make and
 automatically rebuild the site. It will then live reload any open browser pages
 and push the latest content to them. As most Hugo sites are built in a fraction
 of a second, you will be able to save and see your changes nearly instantly.`,
-	//RunE: server,
+		RunE: cc.server,
+	})
+
+	cc.cmd.Flags().IntVarP(&cc.serverPort, "port", "p", 1313, "port on which the server will listen")
+	cc.cmd.Flags().IntVar(&cc.liveReloadPort, "liveReloadPort", -1, "port for live reloading (i.e. 443 in HTTPS proxy situations)")
+	cc.cmd.Flags().StringVarP(&cc.serverInterface, "bind", "", "127.0.0.1", "interface to which the server will bind")
+	cc.cmd.Flags().BoolVarP(&cc.serverWatch, "watch", "w", true, "watch filesystem for changes and recreate as needed")
+	cc.cmd.Flags().BoolVar(&cc.noHTTPCache, "noHTTPCache", false, "prevent HTTP caching")
+	cc.cmd.Flags().BoolVarP(&cc.serverAppend, "appendPort", "", true, "append port to baseURL")
+	cc.cmd.Flags().BoolVar(&cc.disableLiveReload, "disableLiveReload", false, "watch without enabling live browser reload on rebuild")
+	cc.cmd.Flags().BoolVar(&cc.navigateToChanged, "navigateToChanged", false, "navigate to changed content file on live browser reload")
+	cc.cmd.Flags().BoolVar(&cc.renderToDisk, "renderToDisk", false, "render to Destination path (default is render to memory & serve from there)")
+	cc.cmd.Flags().BoolVar(&cc.disableFastRender, "disableFastRender", false, "enables full re-renders on changes")
+	cc.cmd.Flags().BoolVar(&cc.disableBrowserError, "disableBrowserError", false, "do not show build errors in the browser")
+
+	cc.cmd.Flags().String("memstats", "", "log memory usage to this file")
+	cc.cmd.Flags().String("meminterval", "100ms", "interval to poll memory usage (requires --memstats), valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\".")
+
+	return cc
 }
 
 type filesOnlyFs struct {
@@ -90,114 +127,116 @@ func (f noDirFile) Readdir(count int) ([]os.FileInfo, error) {
 	return nil, nil
 }
 
-func init() {
-	initHugoBuilderFlags(serverCmd)
+var serverPorts []int
 
-	serverCmd.Flags().IntVarP(&serverPort, "port", "p", 1313, "port on which the server will listen")
-	serverCmd.Flags().IntVar(&liveReloadPort, "liveReloadPort", -1, "port for live reloading (i.e. 443 in HTTPS proxy situations)")
-	serverCmd.Flags().StringVarP(&serverInterface, "bind", "", "127.0.0.1", "interface to which the server will bind")
-	serverCmd.Flags().BoolVarP(&serverWatch, "watch", "w", true, "watch filesystem for changes and recreate as needed")
-	serverCmd.Flags().BoolVar(&noHTTPCache, "noHTTPCache", false, "prevent HTTP caching")
-	serverCmd.Flags().BoolVarP(&serverAppend, "appendPort", "", true, "append port to baseURL")
-	serverCmd.Flags().BoolVar(&disableLiveReload, "disableLiveReload", false, "watch without enabling live browser reload on rebuild")
-	serverCmd.Flags().BoolVar(&navigateToChanged, "navigateToChanged", false, "navigate to changed content file on live browser reload")
-	serverCmd.Flags().BoolVar(&renderToDisk, "renderToDisk", false, "render to Destination path (default is render to memory & serve from there)")
-	serverCmd.Flags().BoolVar(&disableFastRender, "disableFastRender", false, "enables full re-renders on changes")
-
-	serverCmd.Flags().String("memstats", "", "log memory usage to this file")
-	serverCmd.Flags().String("meminterval", "100ms", "interval to poll memory usage (requires --memstats), valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\".")
-
-	serverCmd.RunE = server
-
-}
-
-func server(cmd *cobra.Command, args []string) error {
+func (sc *serverCmd) server(cmd *cobra.Command, args []string) error {
 	// If a Destination is provided via flag write to disk
+	destination, _ := cmd.Flags().GetString("destination")
 	if destination != "" {
-		renderToDisk = true
+		sc.renderToDisk = true
 	}
 
+	var serverCfgInit sync.Once
+
 	cfgInit := func(c *commandeer) error {
-		c.Set("renderToMemory", !renderToDisk)
+		c.Set("renderToMemory", !sc.renderToDisk)
 		if cmd.Flags().Changed("navigateToChanged") {
-			c.Set("navigateToChanged", navigateToChanged)
+			c.Set("navigateToChanged", sc.navigateToChanged)
 		}
 		if cmd.Flags().Changed("disableLiveReload") {
-			c.Set("disableLiveReload", disableLiveReload)
+			c.Set("disableLiveReload", sc.disableLiveReload)
 		}
 		if cmd.Flags().Changed("disableFastRender") {
-			c.Set("disableFastRender", disableFastRender)
+			c.Set("disableFastRender", sc.disableFastRender)
 		}
-		if serverWatch {
+		if cmd.Flags().Changed("disableBrowserError") {
+			c.Set("disableBrowserError", sc.disableBrowserError)
+		}
+		if sc.serverWatch {
 			c.Set("watch", true)
 		}
 
-		serverPorts := make([]int, 1)
-
-		if c.languages.IsMultihost() {
-			if !serverAppend {
-				return newSystemError("--appendPort=false not supported when in multihost mode")
-			}
-			serverPorts = make([]int, len(c.languages))
+		// TODO(bep) yes, we should fix.
+		if !c.languagesConfigured {
+			return nil
 		}
 
-		currentServerPort := serverPort
+		var err error
 
-		for i := 0; i < len(serverPorts); i++ {
-			l, err := net.Listen("tcp", net.JoinHostPort(serverInterface, strconv.Itoa(currentServerPort)))
-			if err == nil {
-				l.Close()
-				serverPorts[i] = currentServerPort
-			} else {
-				if i == 0 && serverCmd.Flags().Changed("port") {
-					// port set explicitly by user -- he/she probably meant it!
-					return newSystemErrorF("Server startup failed: %s", err)
+		// We can only do this once.
+		serverCfgInit.Do(func() {
+			serverPorts = make([]int, 1)
+
+			if c.languages.IsMultihost() {
+				if !sc.serverAppend {
+					err = newSystemError("--appendPort=false not supported when in multihost mode")
 				}
-				jww.ERROR.Println("port", serverPort, "already in use, attempting to use an available port")
-				sp, err := helpers.FindAvailablePort()
-				if err != nil {
-					return newSystemError("Unable to find alternative port to use:", err)
-				}
-				serverPorts[i] = sp.Port
+				serverPorts = make([]int, len(c.languages))
 			}
 
-			currentServerPort = serverPorts[i] + 1
-		}
+			currentServerPort := sc.serverPort
+
+			for i := 0; i < len(serverPorts); i++ {
+				l, err := net.Listen("tcp", net.JoinHostPort(sc.serverInterface, strconv.Itoa(currentServerPort)))
+				if err == nil {
+					l.Close()
+					serverPorts[i] = currentServerPort
+				} else {
+					if i == 0 && sc.cmd.Flags().Changed("port") {
+						// port set explicitly by user -- he/she probably meant it!
+						err = newSystemErrorF("Server startup failed: %s", err)
+					}
+					c.logger.FEEDBACK.Println("port", sc.serverPort, "already in use, attempting to use an available port")
+					sp, err := helpers.FindAvailablePort()
+					if err != nil {
+						err = newSystemError("Unable to find alternative port to use:", err)
+					}
+					serverPorts[i] = sp.Port
+				}
+
+				currentServerPort = serverPorts[i] + 1
+			}
+		})
 
 		c.serverPorts = serverPorts
 
-		c.Set("port", serverPort)
-		if liveReloadPort != -1 {
-			c.Set("liveReloadPort", liveReloadPort)
+		c.Set("port", sc.serverPort)
+		if sc.liveReloadPort != -1 {
+			c.Set("liveReloadPort", sc.liveReloadPort)
 		} else {
 			c.Set("liveReloadPort", serverPorts[0])
 		}
 
-		if c.languages.IsMultihost() {
-			for i, language := range c.languages {
-				baseURL, err := fixURL(language, baseURL, serverPorts[i])
-				if err != nil {
-					return err
-				}
+		isMultiHost := c.languages.IsMultihost()
+		for i, language := range c.languages {
+			var serverPort int
+			if isMultiHost {
+				serverPort = serverPorts[i]
+			} else {
+				serverPort = serverPorts[0]
+			}
+
+			baseURL, err := sc.fixURL(language, sc.baseURL, serverPort)
+			if err != nil {
+				return nil
+			}
+			if isMultiHost {
 				language.Set("baseURL", baseURL)
 			}
-		} else {
-			baseURL, err := fixURL(c.Cfg, baseURL, serverPorts[0])
-			if err != nil {
-				return err
+			if i == 0 {
+				c.Set("baseURL", baseURL)
 			}
-			c.Set("baseURL", baseURL)
 		}
 
-		return nil
+		return err
 
 	}
 
 	if err := memStats(); err != nil {
-		jww.ERROR.Println("memstats error:", err)
+		jww.WARN.Println("memstats error:", err)
 	}
 
-	c, err := InitializeConfig(true, cfgInit, serverCmd)
+	c, err := initializeConfig(true, true, &sc.hugoBuilderCommon, sc, cfgInit)
 	if err != nil {
 		return err
 	}
@@ -206,16 +245,12 @@ func server(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	for _, s := range Hugo.Sites {
+	for _, s := range c.hugo.Sites {
 		s.RegisterMediaTypes()
 	}
 
-	if serverWatch {
-		c.watchConfig()
-	}
-
 	// Watch runs its own server as part of the routine
-	if serverWatch {
+	if sc.serverWatch {
 
 		watchDirs, err := c.getDirList()
 		if err != nil {
@@ -241,14 +276,16 @@ func server(cmd *cobra.Command, args []string) error {
 
 	}
 
-	return c.serve()
+	return c.serve(sc)
 
 }
 
 type fileServer struct {
-	baseURLs []string
-	roots    []string
-	c        *commandeer
+	baseURLs      []string
+	roots         []string
+	errorTemplate tpl.Template
+	c             *commandeer
+	s             *serverCmd
 }
 
 func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, error) {
@@ -262,43 +299,72 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, erro
 		publishDir = filepath.Join(publishDir, root)
 	}
 
-	absPublishDir := f.c.PathSpec().AbsPathify(publishDir)
+	absPublishDir := f.c.hugo.PathSpec.AbsPathify(publishDir)
 
 	if i == 0 {
-		if renderToDisk {
+		if f.s.renderToDisk {
 			jww.FEEDBACK.Println("Serving pages from " + absPublishDir)
 		} else {
 			jww.FEEDBACK.Println("Serving pages from memory")
 		}
 	}
 
-	httpFs := afero.NewHttpFs(f.c.Fs.Destination)
+	httpFs := afero.NewHttpFs(f.c.destinationFs)
 	fs := filesOnlyFs{httpFs.Dir(absPublishDir)}
 
-	doLiveReload := !buildWatch && !f.c.Cfg.GetBool("disableLiveReload")
-	fastRenderMode := doLiveReload && !f.c.Cfg.GetBool("disableFastRender")
-
-	if i == 0 && fastRenderMode {
+	if i == 0 && f.c.fastRenderMode {
 		jww.FEEDBACK.Println("Running in Fast Render Mode. For full rebuilds on change: hugo server --disableFastRender")
 	}
 
 	// We're only interested in the path
 	u, err := url.Parse(baseURL)
 	if err != nil {
-		return nil, "", "", fmt.Errorf("Invalid baseURL: %s", err)
+		return nil, "", "", errors.Wrap(err, "Invalid baseURL")
 	}
 
 	decorate := func(h http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			if noHTTPCache {
+			if f.c.showErrorInBrowser {
+				// First check the error state
+				err := f.c.getErrorWithContext()
+				if err != nil {
+					w.WriteHeader(500)
+					var b bytes.Buffer
+					err := f.errorTemplate.Execute(&b, err)
+					if err != nil {
+						f.c.logger.ERROR.Println(err)
+					}
+					port = 1313
+					if !f.c.paused {
+						port = f.c.Cfg.GetInt("liveReloadPort")
+					}
+					fmt.Fprint(w, injectLiveReloadScript(&b, port))
+
+					return
+				}
+			}
+
+			if f.s.noHTTPCache {
 				w.Header().Set("Cache-Control", "no-store, no-cache, must-revalidate, max-age=0")
 				w.Header().Set("Pragma", "no-cache")
 			}
 
-			if fastRenderMode {
+			if f.c.fastRenderMode && f.c.buildErr == nil {
 				p := r.RequestURI
 				if strings.HasSuffix(p, "/") || strings.HasSuffix(p, "html") || strings.HasSuffix(p, "htm") {
+					if !f.c.visitedURLs.Contains(p) {
+						// If not already on stack, re-render that single page.
+						if err := f.c.partialReRender(p); err != nil {
+							f.c.handleBuildErr(err, fmt.Sprintf("Failed to render %q", p))
+							if f.c.showErrorInBrowser {
+								http.Redirect(w, r, p, 301)
+								return
+							}
+						}
+					}
+
 					f.c.visitedURLs.Add(p)
+
 				}
 			}
 			h.ServeHTTP(w, r)
@@ -314,14 +380,19 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, erro
 		mu.Handle(u.Path, http.StripPrefix(u.Path, fileserver))
 	}
 
-	endpoint := net.JoinHostPort(serverInterface, strconv.Itoa(port))
+	endpoint := net.JoinHostPort(f.s.serverInterface, strconv.Itoa(port))
 
 	return mu, u.String(), endpoint, nil
 }
 
-func (c *commandeer) serve() error {
+var logErrorRe = regexp.MustCompile("(?s)ERROR \\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2} ")
 
-	isMultiHost := Hugo.IsMultihost()
+func removeErrorPrefixFromLog(content string) string {
+	return logErrorRe.ReplaceAllLiteralString(content, "")
+}
+func (c *commandeer) serve(s *serverCmd) error {
+
+	isMultiHost := c.hugo.IsMultihost()
 
 	var (
 		baseURLs []string
@@ -329,20 +400,27 @@ func (c *commandeer) serve() error {
 	)
 
 	if isMultiHost {
-		for _, s := range Hugo.Sites {
+		for _, s := range c.hugo.Sites {
 			baseURLs = append(baseURLs, s.BaseURL.String())
 			roots = append(roots, s.Language.Lang)
 		}
 	} else {
-		s := Hugo.Sites[0]
+		s := c.hugo.Sites[0]
 		baseURLs = []string{s.BaseURL.String()}
 		roots = []string{""}
 	}
 
+	templ, err := c.hugo.TextTmpl.Parse("__default_server_error", buildErrorTemplate)
+	if err != nil {
+		return err
+	}
+
 	srv := &fileServer{
-		baseURLs: baseURLs,
-		roots:    roots,
-		c:        c,
+		baseURLs:      baseURLs,
+		roots:         roots,
+		c:             c,
+		s:             s,
+		errorTemplate: templ,
 	}
 
 	doLiveReload := !c.Cfg.GetBool("disableLiveReload")
@@ -354,18 +432,18 @@ func (c *commandeer) serve() error {
 	var sigs = make(chan os.Signal)
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
 
-	for i, _ := range baseURLs {
+	for i := range baseURLs {
 		mu, serverURL, endpoint, err := srv.createEndpoint(i)
 
 		if doLiveReload {
 			mu.HandleFunc("/livereload.js", livereload.ServeJS)
 			mu.HandleFunc("/livereload", livereload.Handler)
 		}
-		jww.FEEDBACK.Printf("Web Server is available at %s (bind address %s)\n", serverURL, serverInterface)
+		jww.FEEDBACK.Printf("Web Server is available at %s (bind address %s)\n", serverURL, s.serverInterface)
 		go func() {
 			err = http.ListenAndServe(endpoint, mu)
 			if err != nil {
-				jww.ERROR.Printf("Error: %s\n", err.Error())
+				c.logger.ERROR.Printf("Error: %s\n", err.Error())
 				os.Exit(1)
 			}
 		}()
@@ -373,14 +451,21 @@ func (c *commandeer) serve() error {
 
 	jww.FEEDBACK.Println("Press Ctrl+C to stop")
 
-	<-sigs
+	if s.stop != nil {
+		select {
+		case <-sigs:
+		case <-s.stop:
+		}
+	} else {
+		<-sigs
+	}
 
 	return nil
 }
 
 // fixURL massages the baseURL into a form needed for serving
 // all pages correctly.
-func fixURL(cfg config.Provider, s string, port int) (string, error) {
+func (sc *serverCmd) fixURL(cfg config.Provider, s string, port int) (string, error) {
 	useLocalhost := false
 	if s == "" {
 		s = cfg.GetString("baseURL")
@@ -415,11 +500,11 @@ func fixURL(cfg config.Provider, s string, port int) (string, error) {
 		u.Host = "localhost"
 	}
 
-	if serverAppend {
+	if sc.serverAppend {
 		if strings.Contains(u.Host, ":") {
 			u.Host, _, err = net.SplitHostPort(u.Host)
 			if err != nil {
-				return "", fmt.Errorf("Failed to split baseURL hostpost: %s", err)
+				return "", errors.Wrap(err, "Failed to split baseURL hostpost")
 			}
 		}
 		u.Host += fmt.Sprintf(":%d", port)
@@ -429,9 +514,11 @@ func fixURL(cfg config.Provider, s string, port int) (string, error) {
 }
 
 func memStats() error {
-	memstats := serverCmd.Flags().Lookup("memstats").Value.String()
+	b := newCommandsBuilder()
+	sc := b.newServerCmd().getCommand()
+	memstats := sc.Flags().Lookup("memstats").Value.String()
 	if memstats != "" {
-		interval, err := time.ParseDuration(serverCmd.Flags().Lookup("meminterval").Value.String())
+		interval, err := time.ParseDuration(sc.Flags().Lookup("meminterval").Value.String())
 		if err != nil {
 			interval, _ = time.ParseDuration("100ms")
 		}

commands/server_errors.go

@@ -0,0 +1,95 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package commands
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/gohugoio/hugo/transform"
+	"github.com/gohugoio/hugo/transform/livereloadinject"
+)
+
+var buildErrorTemplate = `<!doctype html>
+<html class="no-js" lang="">
+	<head>
+		<meta charset="utf-8">
+		<title>Hugo Server: Error</title>
+		<style type="text/css">
+		body {
+			font-family: "Muli",avenir, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
+			font-size: 16px;
+			background-color: black;
+			color: rgba(255, 255, 255, 0.9);
+		}
+		main {
+			margin: auto;
+			width: 95%;
+			padding: 1rem;
+		}		
+		.version {
+			color: #ccc;
+			padding: 1rem 0;
+		}
+		.stack {
+			margin-top: 6rem;
+		}
+		pre {
+			white-space: pre-wrap;      
+			white-space: -moz-pre-wrap;  
+			white-space: -pre-wrap;     
+			white-space: -o-pre-wrap;    
+			word-wrap: break-word;     
+		}
+		.highlight {
+			overflow-x: auto;
+			padding: 0.75rem;
+			margin-bottom: 1rem;
+			background-color: #272822;
+			border: 1px solid black;
+		}
+		a {
+			color: #0594cb;
+			text-decoration: none;
+		}
+		a:hover {
+			color: #ccc;
+		}
+		</style>
+	</head>
+	<body>
+		<main>
+			{{ highlight .Error "apl" "noclasses=true,style=monokai" }}
+			{{ with .File }}
+			{{ $params := printf "noclasses=true,style=monokai,linenos=table,hl_lines=%d,linenostart=%d" (add .LinesPos 1) (sub .Position.LineNumber .LinesPos) }}
+			{{ $lexer := .ChromaLexer | default "go-html-template" }}
+			{{  highlight (delimit .Lines "\n") $lexer $params }}
+			{{ end }}
+			{{ with .StackTrace }}
+			{{ highlight . "apl" "noclasses=true,style=monokai" }}
+			{{ end }}
+			<p class="version">{{ .Version }}</p>
+			<a href="">Reload Page</a>
+		</main>
+</body>
+</html>
+`
+
+func injectLiveReloadScript(src io.Reader, port int) string {
+	var b bytes.Buffer
+	chain := transform.Chain{livereloadinject.New(port)}
+	chain.Apply(&b, src)
+
+	return b.String()
+}

commands/server_test.go

@@ -14,11 +14,66 @@
 package commands
 
 import (
+	"fmt"
+	"net/http"
+	"os"
+	"runtime"
+	"strings"
 	"testing"
+	"time"
+
+	"github.com/gohugoio/hugo/helpers"
 
 	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
 )
 
+func TestServer(t *testing.T) {
+	if isWindowsCI() {
+		// TODO(bep) not sure why server tests have started to fail on the Windows CI server.
+		t.Skip("Skip server test on appveyor")
+	}
+	assert := require.New(t)
+	dir, err := createSimpleTestSite(t)
+	assert.NoError(err)
+
+	// Let us hope that this port is available on all systems ...
+	port := 1331
+
+	defer func() {
+		os.RemoveAll(dir)
+	}()
+
+	stop := make(chan bool)
+
+	b := newCommandsBuilder()
+	scmd := b.newServerCmdSignaled(stop)
+
+	cmd := scmd.getCommand()
+	cmd.SetArgs([]string{"-s=" + dir, fmt.Sprintf("-p=%d", port)})
+
+	go func() {
+		_, err = cmd.ExecuteC()
+		assert.NoError(err)
+	}()
+
+	// There is no way to know exactly when the server is ready for connections.
+	// We could improve by something like https://golang.org/pkg/net/http/httptest/#Server
+	// But for now, let us sleep and pray!
+	time.Sleep(2 * time.Second)
+
+	resp, err := http.Get("http://localhost:1331/")
+	assert.NoError(err)
+	defer resp.Body.Close()
+	homeContent := helpers.ReaderToString(resp.Body)
+
+	assert.Contains(homeContent, "List: Hugo Commands")
+
+	// Stop the server.
+	stop <- true
+
+}
+
 func TestFixURL(t *testing.T) {
 	type data struct {
 		TestName   string
@@ -42,12 +97,14 @@ func TestFixURL(t *testing.T) {
 	}
 
 	for i, test := range tests {
+		b := newCommandsBuilder()
+		s := b.newServerCmd()
 		v := viper.New()
-		baseURL = test.CLIBaseURL
+		baseURL := test.CLIBaseURL
 		v.Set("baseURL", test.CfgBaseURL)
-		serverAppend = test.AppendPort
-		serverPort = test.Port
-		result, err := fixURL(v, baseURL, serverPort)
+		s.serverAppend = test.AppendPort
+		s.serverPort = test.Port
+		result, err := s.fixURL(v, baseURL, s.serverPort)
 		if err != nil {
 			t.Errorf("Test #%d %s: unexpected error %s", i, test.TestName, err)
 		}
@@ -56,3 +113,19 @@ func TestFixURL(t *testing.T) {
 		}
 	}
 }
+
+func TestRemoveErrorPrefixFromLog(t *testing.T) {
+	assert := require.New(t)
+	content := `ERROR 2018/10/07 13:11:12 Error while rendering "home": template: _default/baseof.html:4:3: executing "main" at <partial "logo" .>: error calling partial: template: partials/logo.html:5:84: executing "partials/logo.html" at <$resized.AHeight>: can't evaluate field AHeight in type *resource.Image
+ERROR 2018/10/07 13:11:12 Rebuild failed: logged 1 error(s)
+`
+
+	withoutError := removeErrorPrefixFromLog(content)
+
+	assert.False(strings.Contains(withoutError, "ERROR"), withoutError)
+
+}
+
+func isWindowsCI() bool {
+	return runtime.GOOS == "windows" && os.Getenv("CI") != ""
+}

commands/static_syncer.go

@@ -17,53 +17,43 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/gohugoio/hugo/hugolib/filesystems"
+
 	"github.com/fsnotify/fsnotify"
 	"github.com/gohugoio/hugo/helpers"
-	src "github.com/gohugoio/hugo/source"
 	"github.com/spf13/fsync"
 )
 
 type staticSyncer struct {
 	c *commandeer
-	d *src.Dirs
 }
 
 func newStaticSyncer(c *commandeer) (*staticSyncer, error) {
-	dirs, err := src.NewDirs(c.Fs, c.Cfg, c.DepsCfg.Logger)
-	if err != nil {
-		return nil, err
-	}
-
-	return &staticSyncer{c: c, d: dirs}, nil
+	return &staticSyncer{c: c}, nil
 }
 
-func (s *staticSyncer) isStatic(path string) bool {
-	return s.d.IsStatic(path)
+func (s *staticSyncer) isStatic(filename string) bool {
+	return s.c.hugo.BaseFs.SourceFilesystems.IsStatic(filename)
 }
 
 func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 	c := s.c
 
-	syncFn := func(dirs *src.Dirs, publishDir string) (uint64, error) {
-		staticSourceFs, err := dirs.CreateStaticFs()
-		if err != nil {
-			return 0, err
+	syncFn := func(sourceFs *filesystems.SourceFilesystem) (uint64, error) {
+		publishDir := c.hugo.PathSpec.PublishDir
+		// If root, remove the second '/'
+		if publishDir == "//" {
+			publishDir = helpers.FilePathSeparator
 		}
 
-		if dirs.Language != nil {
-			// Multihost setup
-			publishDir = filepath.Join(publishDir, dirs.Language.Lang)
-		}
-
-		if staticSourceFs == nil {
-			c.Logger.WARN.Println("No static directories found to sync")
-			return 0, nil
+		if sourceFs.PublishFolder != "" {
+			publishDir = filepath.Join(publishDir, sourceFs.PublishFolder)
 		}
 
 		syncer := fsync.NewSyncer()
 		syncer.NoTimes = c.Cfg.GetBool("noTimes")
 		syncer.NoChmod = c.Cfg.GetBool("noChmod")
-		syncer.SrcFs = staticSourceFs
+		syncer.SrcFs = sourceFs.Fs
 		syncer.DestFs = c.Fs.Destination
 
 		// prevent spamming the log on changes
@@ -88,8 +78,7 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 
 			fromPath := ev.Name
 
-			// If we are here we already know the event took place in a static dir
-			relPath := dirs.MakeStaticPathRelative(fromPath)
+			relPath := sourceFs.MakePathRelative(fromPath)
 			if relPath == "" {
 				// Not member of this virtual host.
 				continue
@@ -105,7 +94,7 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 			// the source of that static file. In this case Hugo will incorrectly remove that file
 			// from the published directory.
 			if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove {
-				if _, err := staticSourceFs.Stat(relPath); os.IsNotExist(err) {
+				if _, err := sourceFs.Fs.Stat(relPath); os.IsNotExist(err) {
 					// If file doesn't exist in any static dir, remove it
 					toRemove := filepath.Join(publishDir, relPath)
 
@@ -116,10 +105,10 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 					logger.Println("Syncing", relPath, "to", publishDir)
 
 					if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil {
-						c.Logger.ERROR.Println(err)
+						c.logger.ERROR.Println(err)
 					}
 				} else {
-					c.Logger.ERROR.Println(err)
+					c.logger.ERROR.Println(err)
 				}
 
 				continue
@@ -128,7 +117,7 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 			// For all other event operations Hugo will sync static.
 			logger.Println("Syncing", relPath, "to", publishDir)
 			if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil {
-				c.Logger.ERROR.Println(err)
+				c.logger.ERROR.Println(err)
 			}
 		}
 

commands/undraft.go

@@ -1,157 +0,0 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package commands
-
-import (
-	"bytes"
-	"errors"
-	"os"
-	"time"
-
-	"github.com/gohugoio/hugo/parser"
-	"github.com/spf13/cobra"
-)
-
-var undraftCmd = &cobra.Command{
-	Use:   "undraft path/to/content",
-	Short: "Undraft resets the content's draft status",
-	Long: `Undraft resets the content's draft status
-and updates the date to the current date and time.
-If the content's draft status is 'False', nothing is done.`,
-	RunE: Undraft,
-}
-
-// Undraft publishes the specified content by setting its draft status
-// to false and setting its publish date to now. If the specified content is
-// not a draft, it will log an error.
-func Undraft(cmd *cobra.Command, args []string) error {
-	c, err := InitializeConfig(false, nil)
-
-	if err != nil {
-		return err
-	}
-
-	if len(args) < 1 {
-		return newUserError("a piece of content needs to be specified")
-	}
-
-	cfg := c.DepsCfg
-
-	location := args[0]
-	// open the file
-	f, err := cfg.Fs.Source.Open(location)
-	if err != nil {
-		return err
-	}
-
-	// get the page from file
-	p, err := parser.ReadFrom(f)
-	f.Close()
-	if err != nil {
-		return err
-	}
-
-	w, err := undraftContent(p)
-	if err != nil {
-		return newSystemErrorF("an error occurred while undrafting %q: %s", location, err)
-	}
-
-	f, err = cfg.Fs.Source.OpenFile(location, os.O_WRONLY|os.O_TRUNC, 0644)
-	if err != nil {
-		return newSystemErrorF("%q not be undrafted due to error opening file to save changes: %q\n", location, err)
-	}
-	defer f.Close()
-	_, err = w.WriteTo(f)
-	if err != nil {
-		return newSystemErrorF("%q not be undrafted due to save error: %q\n", location, err)
-	}
-	return nil
-}
-
-// undraftContent: if the content is a draft, change its draft status to
-// 'false' and set the date to time.Now(). If the draft status is already
-// 'false', don't do anything.
-func undraftContent(p parser.Page) (bytes.Buffer, error) {
-	var buff bytes.Buffer
-	// get the metadata; easiest way to see if it's a draft
-	meta, err := p.Metadata()
-	if err != nil {
-		return buff, err
-	}
-	// since the metadata was obtainable, we can also get the key/value separator for
-	// Front Matter
-	fm := p.FrontMatter()
-	if fm == nil {
-		return buff, errors.New("Front Matter was found, nothing was finalized")
-	}
-
-	var isDraft, gotDate bool
-	var date string
-L:
-	for k, v := range meta.(map[string]interface{}) {
-		switch k {
-		case "draft":
-			if !v.(bool) {
-				return buff, errors.New("not a Draft: nothing was done")
-			}
-			isDraft = true
-			if gotDate {
-				break L
-			}
-		case "date":
-			date = v.(string) // capture the value to make replacement easier
-			gotDate = true
-			if isDraft {
-				break L
-			}
-		}
-	}
-
-	// if draft wasn't found in FrontMatter, it isn't a draft.
-	if !isDraft {
-		return buff, errors.New("not a Draft: nothing was done")
-	}
-
-	// get the front matter as bytes and split it into lines
-	var lineEnding []byte
-	fmLines := bytes.Split(fm, []byte("\n"))
-	if len(fmLines) == 1 { // if the result is only 1 element, try to split on dos line endings
-		fmLines = bytes.Split(fm, []byte("\r\n"))
-		if len(fmLines) == 1 {
-			return buff, errors.New("unable to split FrontMatter into lines")
-		}
-		lineEnding = append(lineEnding, []byte("\r\n")...)
-	} else {
-		lineEnding = append(lineEnding, []byte("\n")...)
-	}
-
-	// Write the front matter lines to the buffer, replacing as necessary
-	for _, v := range fmLines {
-		pos := bytes.Index(v, []byte("draft"))
-		if pos != -1 {
-			continue
-		}
-		pos = bytes.Index(v, []byte("date"))
-		if pos != -1 { // if date field wasn't found, add it
-			v = bytes.Replace(v, []byte(date), []byte(time.Now().Format(time.RFC3339)), 1)
-		}
-		buff.Write(v)
-		buff.Write(lineEnding)
-	}
-
-	// append the actual content
-	buff.Write(p.Content())
-
-	return buff, nil
-}

commands/undraft_test.go

@@ -1,87 +0,0 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package commands
-
-// TODO Support Mac Encoding (\r)
-
-import (
-	"bytes"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/gohugoio/hugo/parser"
-)
-
-var (
-	jsonFM         = "{\n \"date\": \"12-04-06\",\n \"title\": \"test json\"\n}"
-	jsonDraftFM    = "{\n \"draft\": true,\n \"date\": \"12-04-06\",\n \"title\":\"test json\"\n}"
-	tomlFM         = "+++\n date= \"12-04-06\"\n title= \"test toml\"\n+++"
-	tomlDraftFM    = "+++\n draft= true\n date= \"12-04-06\"\n title=\"test toml\"\n+++"
-	yamlFM         = "---\n date: \"12-04-06\"\n title: \"test yaml\"\n---"
-	yamlDraftFM    = "---\n draft: true\n date: \"12-04-06\"\n title: \"test yaml\"\n---"
-	yamlYesDraftFM = "---\n draft: yes\n date: \"12-04-06\"\n title: \"test yaml\"\n---"
-)
-
-func TestUndraftContent(t *testing.T) {
-	tests := []struct {
-		fm          string
-		expectedErr string
-	}{
-		{jsonFM, "not a Draft: nothing was done"},
-		{jsonDraftFM, ""},
-		{tomlFM, "not a Draft: nothing was done"},
-		{tomlDraftFM, ""},
-		{yamlFM, "not a Draft: nothing was done"},
-		{yamlDraftFM, ""},
-		{yamlYesDraftFM, ""},
-	}
-
-	for i, test := range tests {
-		r := bytes.NewReader([]byte(test.fm))
-		p, _ := parser.ReadFrom(r)
-		res, err := undraftContent(p)
-		if test.expectedErr != "" {
-			if err == nil {
-				t.Errorf("[%d] Expected error, got none", i)
-				continue
-			}
-			if err.Error() != test.expectedErr {
-				t.Errorf("[%d] Expected %q, got %q", i, test.expectedErr, err)
-				continue
-			}
-		} else {
-			r = bytes.NewReader(res.Bytes())
-			p, _ = parser.ReadFrom(r)
-			meta, err := p.Metadata()
-			if err != nil {
-				t.Errorf("[%d] unexpected error %q", i, err)
-				continue
-			}
-			for k, v := range meta.(map[string]interface{}) {
-				if k == "draft" {
-					if v.(bool) {
-						t.Errorf("[%d] Expected %q to be \"false\", got \"true\"", i, k)
-						continue
-					}
-				}
-				if k == "date" {
-					if !strings.HasPrefix(v.(string), time.Now().Format("2006-01-02")) {
-						t.Errorf("[%d] Expected %v to start with %v", i, v.(string), time.Now().Format("2006-01-02"))
-					}
-				}
-			}
-		}
-	}
-}

commands/version.go

@@ -14,29 +14,62 @@
 package commands
 
 import (
+	"fmt"
 	"runtime"
 	"strings"
 
+	jww "github.com/spf13/jwalterweatherman"
+
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugolib"
+	"github.com/gohugoio/hugo/resource/tocss/scss"
 	"github.com/spf13/cobra"
-	jww "github.com/spf13/jwalterweatherman"
 )
 
-var versionCmd = &cobra.Command{
-	Use:   "version",
-	Short: "Print the version number of Hugo",
-	Long:  `All software has versions. This is Hugo's.`,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		printHugoVersion()
-		return nil
-	},
+var _ cmder = (*versionCmd)(nil)
+
+type versionCmd struct {
+	*baseCmd
+}
+
+func newVersionCmd() *versionCmd {
+	return &versionCmd{
+		newBaseCmd(&cobra.Command{
+			Use:   "version",
+			Short: "Print the version number of Hugo",
+			Long:  `All software has versions. This is Hugo's.`,
+			RunE: func(cmd *cobra.Command, args []string) error {
+				printHugoVersion()
+				return nil
+			},
+		}),
+	}
 }
 
 func printHugoVersion() {
-	if hugolib.CommitHash == "" {
-		jww.FEEDBACK.Printf("Hugo Static Site Generator v%s %s/%s BuildDate: %s\n", helpers.CurrentHugoVersion, runtime.GOOS, runtime.GOARCH, hugolib.BuildDate)
-	} else {
-		jww.FEEDBACK.Printf("Hugo Static Site Generator v%s-%s %s/%s BuildDate: %s\n", helpers.CurrentHugoVersion, strings.ToUpper(hugolib.CommitHash), runtime.GOOS, runtime.GOARCH, hugolib.BuildDate)
-	}
+	jww.FEEDBACK.Println(hugoVersionString())
+}
+
+func hugoVersionString() string {
+	program := "Hugo Static Site Generator"
+
+	version := "v" + helpers.CurrentHugoVersion.String()
+	if hugolib.CommitHash != "" {
+		version += "-" + strings.ToUpper(hugolib.CommitHash)
+	}
+	if scss.Supports() {
+		version += "/extended"
+	}
+
+	osArch := runtime.GOOS + "/" + runtime.GOARCH
+
+	var buildDate string
+	if hugolib.BuildDate != "" {
+		buildDate = hugolib.BuildDate
+	} else {
+		buildDate = "unknown"
+	}
+
+	return fmt.Sprintf("%s %s %s BuildDate: %s", program, version, osArch, buildDate)
+
 }

common/collections/append.go

@@ -0,0 +1,114 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collections
+
+import (
+	"fmt"
+	"reflect"
+)
+
+// Append appends from to a slice to and returns the resulting slice.
+// If length of from is one and the only element is a slice of same type as to,
+// it will be appended.
+func Append(to interface{}, from ...interface{}) (interface{}, error) {
+	tov, toIsNil := indirect(reflect.ValueOf(to))
+
+	toIsNil = toIsNil || to == nil
+	var tot reflect.Type
+
+	if !toIsNil {
+		if tov.Kind() != reflect.Slice {
+			return nil, fmt.Errorf("expected a slice, got %T", to)
+		}
+
+		tot = tov.Type().Elem()
+		toIsNil = tov.Len() == 0
+
+		if len(from) == 1 {
+			fromv := reflect.ValueOf(from[0])
+			if fromv.Kind() == reflect.Slice {
+				if toIsNil {
+					// If we get nil []string, we just return the []string
+					return from[0], nil
+				}
+
+				fromt := reflect.TypeOf(from[0]).Elem()
+
+				// If we get []string []string, we append the from slice to to
+				if tot == fromt {
+					return reflect.AppendSlice(tov, fromv).Interface(), nil
+				} else if !fromt.AssignableTo(tot) {
+					// Fall back to a []interface{} slice.
+					return appendToInterfaceSliceFromValues(tov, fromv)
+
+				}
+			}
+		}
+	}
+
+	if toIsNil {
+		return Slice(from...), nil
+	}
+
+	for _, f := range from {
+		fv := reflect.ValueOf(f)
+		if !fv.Type().AssignableTo(tot) {
+			// Fall back to a []interface{} slice.
+			return appendToInterfaceSlice(tov, from...)
+		}
+		tov = reflect.Append(tov, fv)
+	}
+
+	return tov.Interface(), nil
+}
+
+func appendToInterfaceSliceFromValues(slice1, slice2 reflect.Value) ([]interface{}, error) {
+	var tos []interface{}
+
+	for _, slice := range []reflect.Value{slice1, slice2} {
+		for i := 0; i < slice.Len(); i++ {
+			tos = append(tos, slice.Index(i).Interface())
+		}
+	}
+
+	return tos, nil
+}
+
+func appendToInterfaceSlice(tov reflect.Value, from ...interface{}) ([]interface{}, error) {
+	var tos []interface{}
+
+	for i := 0; i < tov.Len(); i++ {
+		tos = append(tos, tov.Index(i).Interface())
+	}
+
+	for _, v := range from {
+		tos = append(tos, v)
+	}
+
+	return tos, nil
+}
+
+// indirect is borrowed from the Go stdlib: 'text/template/exec.go'
+// TODO(bep) consolidate
+func indirect(v reflect.Value) (rv reflect.Value, isNil bool) {
+	for ; v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface; v = v.Elem() {
+		if v.IsNil() {
+			return v, true
+		}
+		if v.Kind() == reflect.Interface && v.NumMethod() > 0 {
+			break
+		}
+	}
+	return v, false
+}

common/collections/append_test.go

@@ -0,0 +1,78 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collections
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestAppend(t *testing.T) {
+	t.Parallel()
+
+	for i, test := range []struct {
+		start    interface{}
+		addend   []interface{}
+		expected interface{}
+	}{
+		{[]string{"a", "b"}, []interface{}{"c"}, []string{"a", "b", "c"}},
+		{[]string{"a", "b"}, []interface{}{"c", "d", "e"}, []string{"a", "b", "c", "d", "e"}},
+		{[]string{"a", "b"}, []interface{}{[]string{"c", "d", "e"}}, []string{"a", "b", "c", "d", "e"}},
+		{nil, []interface{}{"a", "b"}, []string{"a", "b"}},
+		{nil, []interface{}{nil}, []interface{}{nil}},
+		{[]interface{}{}, []interface{}{[]string{"c", "d", "e"}}, []string{"c", "d", "e"}},
+		{tstSlicers{&tstSlicer{"a"}, &tstSlicer{"b"}},
+			[]interface{}{&tstSlicer{"c"}},
+			tstSlicers{&tstSlicer{"a"}, &tstSlicer{"b"}, &tstSlicer{"c"}}},
+		{&tstSlicers{&tstSlicer{"a"}, &tstSlicer{"b"}},
+			[]interface{}{&tstSlicer{"c"}},
+			tstSlicers{&tstSlicer{"a"},
+				&tstSlicer{"b"},
+				&tstSlicer{"c"}}},
+		{testSlicerInterfaces{&tstSlicerIn1{"a"}, &tstSlicerIn1{"b"}},
+			[]interface{}{&tstSlicerIn1{"c"}},
+			testSlicerInterfaces{&tstSlicerIn1{"a"}, &tstSlicerIn1{"b"}, &tstSlicerIn1{"c"}}},
+		//https://github.com/gohugoio/hugo/issues/5361
+		{[]string{"a", "b"}, []interface{}{tstSlicers{&tstSlicer{"a"}, &tstSlicer{"b"}}},
+			[]interface{}{"a", "b", &tstSlicer{"a"}, &tstSlicer{"b"}}},
+		{[]string{"a", "b"}, []interface{}{&tstSlicer{"a"}},
+			[]interface{}{"a", "b", &tstSlicer{"a"}}},
+		// Errors
+		{"", []interface{}{[]string{"a", "b"}}, false},
+		// No string concatenation.
+		{"ab",
+			[]interface{}{"c"},
+			false},
+	} {
+
+		errMsg := fmt.Sprintf("[%d]", i)
+
+		result, err := Append(test.start, test.addend...)
+
+		if b, ok := test.expected.(bool); ok && !b {
+			require.Error(t, err, errMsg)
+			continue
+		}
+
+		require.NoError(t, err, errMsg)
+
+		if !reflect.DeepEqual(test.expected, result) {
+			t.Fatalf("%s got\n%T: %v\nexpected\n%T: %v", errMsg, result, result, test.expected, test.expected)
+		}
+	}
+
+}

common/collections/collections.go

@@ -0,0 +1,21 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package collections contains common Hugo functionality related to collection
+// handling.
+package collections
+
+// Grouper defines a very generic way to group items by a given key.
+type Grouper interface {
+	Group(key interface{}, items interface{}) (interface{}, error)
+}

common/collections/slice.go

@@ -0,0 +1,66 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collections
+
+import (
+	"reflect"
+)
+
+// Slicer definse a very generic way to create a typed slice. This is used
+// in collections.Slice template func to get types such as Pages, PageGroups etc.
+// instead of the less useful []interface{}.
+type Slicer interface {
+	Slice(items interface{}) (interface{}, error)
+}
+
+// Slice returns a slice of all passed arguments.
+func Slice(args ...interface{}) interface{} {
+	if len(args) == 0 {
+		return args
+	}
+
+	first := args[0]
+	firstType := reflect.TypeOf(first)
+
+	if firstType == nil {
+		return args
+	}
+
+	if g, ok := first.(Slicer); ok {
+		v, err := g.Slice(args)
+		if err == nil {
+			return v
+		}
+
+		// If Slice fails, the items are not of the same type and
+		// []interface{} is the best we can do.
+		return args
+	}
+
+	if len(args) > 1 {
+		// This can be a mix of types.
+		for i := 1; i < len(args); i++ {
+			if firstType != reflect.TypeOf(args[i]) {
+				// []interface{} is the best we can do
+				return args
+			}
+		}
+	}
+
+	slice := reflect.MakeSlice(reflect.SliceOf(firstType), len(args), len(args))
+	for i, arg := range args {
+		slice.Index(i).Set(reflect.ValueOf(arg))
+	}
+	return slice.Interface()
+}

common/collections/slice_test.go

@@ -0,0 +1,125 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collections
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/alecthomas/assert"
+)
+
+var _ Slicer = (*tstSlicer)(nil)
+var _ Slicer = (*tstSlicerIn1)(nil)
+var _ Slicer = (*tstSlicerIn2)(nil)
+var _ testSlicerInterface = (*tstSlicerIn1)(nil)
+var _ testSlicerInterface = (*tstSlicerIn1)(nil)
+
+type testSlicerInterface interface {
+	Name() string
+}
+
+type testSlicerInterfaces []testSlicerInterface
+
+type tstSlicerIn1 struct {
+	name string
+}
+
+type tstSlicerIn2 struct {
+	name string
+}
+
+type tstSlicer struct {
+	name string
+}
+
+func (p *tstSlicerIn1) Slice(in interface{}) (interface{}, error) {
+	items := in.([]interface{})
+	result := make(testSlicerInterfaces, len(items))
+	for i, v := range items {
+		switch vv := v.(type) {
+		case testSlicerInterface:
+			result[i] = vv
+		default:
+			return nil, errors.New("invalid type")
+		}
+
+	}
+	return result, nil
+}
+
+func (p *tstSlicerIn2) Slice(in interface{}) (interface{}, error) {
+	items := in.([]interface{})
+	result := make(testSlicerInterfaces, len(items))
+	for i, v := range items {
+		switch vv := v.(type) {
+		case testSlicerInterface:
+			result[i] = vv
+		default:
+			return nil, errors.New("invalid type")
+		}
+	}
+	return result, nil
+}
+
+func (p *tstSlicerIn1) Name() string {
+	return p.Name()
+}
+
+func (p *tstSlicerIn2) Name() string {
+	return p.Name()
+}
+
+func (p *tstSlicer) Slice(in interface{}) (interface{}, error) {
+	items := in.([]interface{})
+	result := make(tstSlicers, len(items))
+	for i, v := range items {
+		switch vv := v.(type) {
+		case *tstSlicer:
+			result[i] = vv
+		default:
+			return nil, errors.New("invalid type")
+		}
+	}
+	return result, nil
+}
+
+type tstSlicers []*tstSlicer
+
+func TestSlice(t *testing.T) {
+	t.Parallel()
+
+	for i, test := range []struct {
+		args     []interface{}
+		expected interface{}
+	}{
+		{[]interface{}{"a", "b"}, []string{"a", "b"}},
+		{[]interface{}{&tstSlicer{"a"}, &tstSlicer{"b"}}, tstSlicers{&tstSlicer{"a"}, &tstSlicer{"b"}}},
+		{[]interface{}{&tstSlicer{"a"}, "b"}, []interface{}{&tstSlicer{"a"}, "b"}},
+		{[]interface{}{}, []interface{}{}},
+		{[]interface{}{nil}, []interface{}{nil}},
+		{[]interface{}{5, "b"}, []interface{}{5, "b"}},
+		{[]interface{}{&tstSlicerIn1{"a"}, &tstSlicerIn2{"b"}}, testSlicerInterfaces{&tstSlicerIn1{"a"}, &tstSlicerIn2{"b"}}},
+		{[]interface{}{&tstSlicerIn1{"a"}, &tstSlicer{"b"}}, []interface{}{&tstSlicerIn1{"a"}, &tstSlicer{"b"}}},
+	} {
+		errMsg := fmt.Sprintf("[%d] %v", i, test.args)
+
+		result := Slice(test.args...)
+
+		assert.Equal(t, test.expected, result, errMsg)
+	}
+
+	assert.Len(t, Slice(), 0)
+}

common/herrors/error_locator.go

@@ -0,0 +1,251 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package errors contains common Hugo errors and error related utilities.
+package herrors
+
+import (
+	"io"
+	"io/ioutil"
+	"strings"
+
+	"github.com/gohugoio/hugo/common/text"
+	"github.com/gohugoio/hugo/helpers"
+
+	"github.com/spf13/afero"
+)
+
+// LineMatcher contains the elements used to match an error to a line
+type LineMatcher struct {
+	Position text.Position
+	Error    error
+
+	LineNumber int
+	Offset     int
+	Line       string
+}
+
+// LineMatcherFn is used to match a line with an error.
+type LineMatcherFn func(m LineMatcher) bool
+
+// SimpleLineMatcher simply matches by line number.
+var SimpleLineMatcher = func(m LineMatcher) bool {
+	return m.Position.LineNumber == m.LineNumber
+}
+
+var _ text.Positioner = ErrorContext{}
+
+// ErrorContext contains contextual information about an error. This will
+// typically be the lines surrounding some problem in a file.
+type ErrorContext struct {
+
+	// If a match will contain the matched line and up to 2 lines before and after.
+	// Will be empty if no match.
+	Lines []string
+
+	// The position of the error in the Lines above. 0 based.
+	LinesPos int
+
+	position text.Position
+
+	// The lexer to use for syntax highlighting.
+	// https://gohugo.io/content-management/syntax-highlighting/#list-of-chroma-highlighting-languages
+	ChromaLexer string
+}
+
+// Position returns the text position of this error.
+func (e ErrorContext) Position() text.Position {
+	return e.position
+}
+
+var _ causer = (*ErrorWithFileContext)(nil)
+
+// ErrorWithFileContext is an error with some additional file context related
+// to that error.
+type ErrorWithFileContext struct {
+	cause error
+	ErrorContext
+}
+
+func (e *ErrorWithFileContext) Error() string {
+	pos := e.Position()
+	if pos.IsValid() {
+		return pos.String() + ": " + e.cause.Error()
+	}
+	return e.cause.Error()
+}
+
+func (e *ErrorWithFileContext) Cause() error {
+	return e.cause
+}
+
+// WithFileContextForFile will try to add a file context with lines matching the given matcher.
+// If no match could be found, the original error is returned with false as the second return value.
+func WithFileContextForFile(e error, realFilename, filename string, fs afero.Fs, matcher LineMatcherFn) (error, bool) {
+	f, err := fs.Open(filename)
+	if err != nil {
+		return e, false
+	}
+	defer f.Close()
+	return WithFileContext(e, realFilename, f, matcher)
+}
+
+// WithFileContextForFile will try to add a file context with lines matching the given matcher.
+// If no match could be found, the original error is returned with false as the second return value.
+func WithFileContext(e error, realFilename string, r io.Reader, matcher LineMatcherFn) (error, bool) {
+	if e == nil {
+		panic("error missing")
+	}
+	le := UnwrapFileError(e)
+
+	if le == nil {
+		var ok bool
+		if le, ok = ToFileError("", e).(FileError); !ok {
+			return e, false
+		}
+	}
+
+	var errCtx ErrorContext
+
+	posle := le.Position()
+
+	if posle.Offset != -1 {
+		errCtx = locateError(r, le, func(m LineMatcher) bool {
+			if posle.Offset >= m.Offset && posle.Offset < m.Offset+len(m.Line) {
+				lno := posle.LineNumber - m.Position.LineNumber + m.LineNumber
+				m.Position = text.Position{LineNumber: lno}
+			}
+			return matcher(m)
+		})
+	} else {
+		errCtx = locateError(r, le, matcher)
+	}
+
+	pos := &errCtx.position
+
+	if pos.LineNumber == -1 {
+		return e, false
+	}
+
+	pos.Filename = realFilename
+
+	if le.Type() != "" {
+		errCtx.ChromaLexer = chromaLexerFromType(le.Type())
+	} else {
+		errCtx.ChromaLexer = chromaLexerFromFilename(realFilename)
+	}
+
+	return &ErrorWithFileContext{cause: e, ErrorContext: errCtx}, true
+}
+
+// UnwrapErrorWithFileContext tries to unwrap an ErrorWithFileContext from err.
+// It returns nil if this is not possible.
+func UnwrapErrorWithFileContext(err error) *ErrorWithFileContext {
+	for err != nil {
+		switch v := err.(type) {
+		case *ErrorWithFileContext:
+			return v
+		case causer:
+			err = v.Cause()
+		default:
+			return nil
+		}
+	}
+	return nil
+}
+
+func chromaLexerFromType(fileType string) string {
+	switch fileType {
+	case "html", "htm":
+		return "go-html-template"
+	}
+	return fileType
+}
+
+func chromaLexerFromFilename(filename string) string {
+	if strings.Contains(filename, "layouts") {
+		return "go-html-template"
+	}
+
+	ext := helpers.ExtNoDelimiter(filename)
+	return chromaLexerFromType(ext)
+}
+
+func locateErrorInString(src string, matcher LineMatcherFn) ErrorContext {
+	return locateError(strings.NewReader(src), &fileError{}, matcher)
+}
+
+func locateError(r io.Reader, le FileError, matches LineMatcherFn) ErrorContext {
+	if le == nil {
+		panic("must provide an error")
+	}
+
+	errCtx := ErrorContext{position: text.Position{LineNumber: -1, ColumnNumber: 1, Offset: -1}, LinesPos: -1}
+
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		return errCtx
+	}
+
+	pos := &errCtx.position
+	lepos := le.Position()
+
+	lines := strings.Split(string(b), "\n")
+
+	if le != nil && lepos.ColumnNumber >= 0 {
+		pos.ColumnNumber = lepos.ColumnNumber
+	}
+
+	lineNo := 0
+	posBytes := 0
+
+	for li, line := range lines {
+		lineNo = li + 1
+		m := LineMatcher{
+			Position:   le.Position(),
+			Error:      le,
+			LineNumber: lineNo,
+			Offset:     posBytes,
+			Line:       line,
+		}
+		if errCtx.LinesPos == -1 && matches(m) {
+			pos.LineNumber = lineNo
+			break
+		}
+
+		posBytes += len(line)
+	}
+
+	if pos.LineNumber != -1 {
+		low := pos.LineNumber - 3
+		if low < 0 {
+			low = 0
+		}
+
+		if pos.LineNumber > 2 {
+			errCtx.LinesPos = 2
+		} else {
+			errCtx.LinesPos = pos.LineNumber - 1
+		}
+
+		high := pos.LineNumber + 2
+		if high > len(lines) {
+			high = len(lines)
+		}
+
+		errCtx.Lines = lines[low:high]
+
+	}
+
+	return errCtx
+}

common/herrors/error_locator_test.go

@@ -0,0 +1,129 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package errors contains common Hugo errors and error related utilities.
+package herrors
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestErrorLocator(t *testing.T) {
+	assert := require.New(t)
+
+	lineMatcher := func(m LineMatcher) bool {
+		return strings.Contains(m.Line, "THEONE")
+	}
+
+	lines := `LINE 1
+LINE 2
+LINE 3
+LINE 4
+This is THEONE
+LINE 6
+LINE 7
+LINE 8
+`
+
+	location := locateErrorInString(lines, lineMatcher)
+	assert.Equal([]string{"LINE 3", "LINE 4", "This is THEONE", "LINE 6", "LINE 7"}, location.Lines)
+
+	pos := location.Position()
+	assert.Equal(5, pos.LineNumber)
+	assert.Equal(2, location.LinesPos)
+
+	assert.Equal([]string{"This is THEONE"}, locateErrorInString(`This is THEONE`, lineMatcher).Lines)
+
+	location = locateErrorInString(`L1
+This is THEONE
+L2
+`, lineMatcher)
+	assert.Equal(2, location.Position().LineNumber)
+	assert.Equal(1, location.LinesPos)
+	assert.Equal([]string{"L1", "This is THEONE", "L2", ""}, location.Lines)
+
+	location = locateErrorInString(`This is THEONE
+L2
+`, lineMatcher)
+	assert.Equal(0, location.LinesPos)
+	assert.Equal([]string{"This is THEONE", "L2", ""}, location.Lines)
+
+	location = locateErrorInString(`L1
+This THEONE
+`, lineMatcher)
+	assert.Equal([]string{"L1", "This THEONE", ""}, location.Lines)
+	assert.Equal(1, location.LinesPos)
+
+	location = locateErrorInString(`L1
+L2
+This THEONE
+`, lineMatcher)
+	assert.Equal([]string{"L1", "L2", "This THEONE", ""}, location.Lines)
+	assert.Equal(2, location.LinesPos)
+
+	location = locateErrorInString("NO MATCH", lineMatcher)
+	assert.Equal(-1, location.Position().LineNumber)
+	assert.Equal(-1, location.LinesPos)
+	assert.Equal(0, len(location.Lines))
+
+	lineMatcher = func(m LineMatcher) bool {
+		return m.LineNumber == 6
+	}
+
+	location = locateErrorInString(`A
+B
+C
+D
+E
+F
+G
+H
+I
+J`, lineMatcher)
+
+	assert.Equal([]string{"D", "E", "F", "G", "H"}, location.Lines)
+	assert.Equal(6, location.Position().LineNumber)
+	assert.Equal(2, location.LinesPos)
+
+	// Test match EOF
+	lineMatcher = func(m LineMatcher) bool {
+		return m.LineNumber == 4
+	}
+
+	location = locateErrorInString(`A
+B
+C
+`, lineMatcher)
+
+	assert.Equal([]string{"B", "C", ""}, location.Lines)
+	assert.Equal(4, location.Position().LineNumber)
+	assert.Equal(2, location.LinesPos)
+
+	offsetMatcher := func(m LineMatcher) bool {
+		return m.Offset == 1
+	}
+
+	location = locateErrorInString(`A
+B
+C
+D
+E`, offsetMatcher)
+
+	assert.Equal([]string{"A", "B", "C", "D"}, location.Lines)
+	assert.Equal(2, location.Position().LineNumber)
+	assert.Equal(1, location.LinesPos)
+
+}

common/herrors/errors.go

@@ -0,0 +1,53 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package herrors contains common Hugo errors and error related utilities.
+package herrors
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+
+	_errors "github.com/pkg/errors"
+)
+
+// As defined in https://godoc.org/github.com/pkg/errors
+type causer interface {
+	Cause() error
+}
+
+type stackTracer interface {
+	StackTrace() _errors.StackTrace
+}
+
+// PrintStackTrace prints the error's stack trace to stdoud.
+func PrintStackTrace(err error) {
+	FprintStackTrace(os.Stdout, err)
+}
+
+// FprintStackTrace prints the error's stack trace to w.
+func FprintStackTrace(w io.Writer, err error) {
+	if err, ok := err.(stackTracer); ok {
+		for _, f := range err.StackTrace() {
+			fmt.Fprintf(w, "%+s:%d\n", f, f)
+		}
+	}
+}
+
+// ErrFeatureNotAvailable denotes that a feature is unavailable.
+//
+// We will, at least to begin with, make some Hugo features (SCSS with libsass) optional,
+// and this error is used to signal those situations.
+var ErrFeatureNotAvailable = errors.New("this feature is not available in your current Hugo version")

common/herrors/file_error.go

@@ -0,0 +1,133 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitatio	ns under the License.
+
+package herrors
+
+import (
+	"encoding/json"
+
+	"github.com/gohugoio/hugo/common/text"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	_ causer = (*fileError)(nil)
+)
+
+// FileError represents an error when handling a file: Parsing a config file,
+// execute a template etc.
+type FileError interface {
+	error
+
+	text.Positioner
+
+	// A string identifying the type of file, e.g. JSON, TOML, markdown etc.
+	Type() string
+}
+
+var _ FileError = (*fileError)(nil)
+
+type fileError struct {
+	position text.Position
+
+	fileType string
+
+	cause error
+}
+
+// Position returns the text position of this error.
+func (e fileError) Position() text.Position {
+	return e.position
+}
+
+func (e *fileError) Type() string {
+	return e.fileType
+}
+
+func (e *fileError) Error() string {
+	if e.cause == nil {
+		return ""
+	}
+	return e.cause.Error()
+}
+
+func (f *fileError) Cause() error {
+	return f.cause
+}
+
+// NewFileError creates a new FileError.
+func NewFileError(fileType string, offset, lineNumber, columnNumber int, err error) FileError {
+	pos := text.Position{Offset: offset, LineNumber: lineNumber, ColumnNumber: columnNumber}
+	return &fileError{cause: err, fileType: fileType, position: pos}
+}
+
+// UnwrapFileError tries to unwrap a FileError from err.
+// It returns nil if this is not possible.
+func UnwrapFileError(err error) FileError {
+	for err != nil {
+		switch v := err.(type) {
+		case FileError:
+			return v
+		case causer:
+			err = v.Cause()
+		default:
+			return nil
+		}
+	}
+	return nil
+}
+
+// ToFileErrorWithOffset will return a new FileError with a line number
+// with the given offset from the original.
+func ToFileErrorWithOffset(fe FileError, offset int) FileError {
+	pos := fe.Position()
+	return ToFileErrorWithLineNumber(fe, pos.LineNumber+offset)
+}
+
+// ToFileErrorWithOffset will return a new FileError with the given line number.
+func ToFileErrorWithLineNumber(fe FileError, lineNumber int) FileError {
+	pos := fe.Position()
+	pos.LineNumber = lineNumber
+	return &fileError{cause: fe, fileType: fe.Type(), position: pos}
+}
+
+// ToFileError will convert the given error to an error supporting
+// the FileError interface.
+func ToFileError(fileType string, err error) FileError {
+	for _, handle := range lineNumberExtractors {
+		lno, col := handle(err)
+		offset, typ := extractOffsetAndType(err)
+		if fileType == "" {
+			fileType = typ
+		}
+
+		if lno > 0 || offset != -1 {
+			return NewFileError(fileType, offset, lno, col, err)
+		}
+	}
+	// Fall back to the pointing to line number 1.
+	return NewFileError(fileType, -1, 1, 1, err)
+}
+
+func extractOffsetAndType(e error) (int, string) {
+	e = errors.Cause(e)
+	switch v := e.(type) {
+	case *json.UnmarshalTypeError:
+		return int(v.Offset), "json"
+	case *json.SyntaxError:
+		return int(v.Offset), "json"
+	default:
+		return -1, ""
+	}
+}

common/herrors/file_error_test.go

@@ -0,0 +1,57 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package herrors
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/pkg/errors"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestToLineNumberError(t *testing.T) {
+	t.Parallel()
+
+	assert := require.New(t)
+
+	for i, test := range []struct {
+		in           error
+		offset       int
+		lineNumber   int
+		columnNumber int
+	}{
+		{errors.New("no line number for you"), 0, 1, 1},
+		{errors.New(`template: _default/single.html:4:15: executing "_default/single.html" at <.Titles>: can't evaluate field Titles in type *hugolib.PageOutput`), 0, 4, 15},
+		{errors.New("parse failed: template: _default/bundle-resource-meta.html:11: unexpected in operand"), 0, 11, 1},
+		{errors.New(`failed:: template: _default/bundle-resource-meta.html:2:7: executing "main" at <.Titles>`), 0, 2, 7},
+		{errors.New("error in front matter: Near line 32 (last key parsed 'title')"), 0, 32, 1},
+		{errors.New(`failed to load translations: (6, 7): was expecting token =, but got "g" instead`), 0, 6, 7},
+	} {
+
+		got := ToFileError("template", test.in)
+
+		errMsg := fmt.Sprintf("[%d][%T]", i, got)
+		le, ok := got.(FileError)
+		assert.True(ok)
+
+		assert.True(ok, errMsg)
+		pos := le.Position()
+		assert.Equal(test.lineNumber, pos.LineNumber, errMsg)
+		assert.Equal(test.columnNumber, pos.ColumnNumber, errMsg)
+		assert.Error(errors.Cause(got))
+	}
+
+}

common/herrors/line_number_extractors.go

@@ -0,0 +1,66 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitatio	ns under the License.
+
+package herrors
+
+import (
+	"regexp"
+	"strconv"
+)
+
+var lineNumberExtractors = []lineNumberExtractor{
+	// Template/shortcode parse errors
+	newLineNumberErrHandlerFromRegexp(".*:(\\d+):(\\d*):"),
+	newLineNumberErrHandlerFromRegexp(".*:(\\d+):"),
+
+	// TOML parse errors
+	newLineNumberErrHandlerFromRegexp(".*Near line (\\d+)(\\s.*)"),
+
+	// YAML parse errors
+	newLineNumberErrHandlerFromRegexp("line (\\d+):"),
+
+	// i18n bundle errors
+	newLineNumberErrHandlerFromRegexp("\\((\\d+),\\s(\\d*)"),
+}
+
+type lineNumberExtractor func(e error) (int, int)
+
+func newLineNumberErrHandlerFromRegexp(expression string) lineNumberExtractor {
+	re := regexp.MustCompile(expression)
+	return extractLineNo(re)
+}
+
+func extractLineNo(re *regexp.Regexp) lineNumberExtractor {
+	return func(e error) (int, int) {
+		if e == nil {
+			panic("no error")
+		}
+		col := 1
+		s := e.Error()
+		m := re.FindStringSubmatch(s)
+		if len(m) >= 2 {
+			lno, _ := strconv.Atoi(m[1])
+			if len(m) > 2 {
+				col, _ = strconv.Atoi(m[2])
+			}
+
+			if col <= 0 {
+				col = 1
+			}
+
+			return lno, col
+		}
+
+		return -1, col
+	}
+}

common/hugio/readers.go

@@ -0,0 +1,53 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugio
+
+import (
+	"io"
+	"strings"
+)
+
+// ReadSeeker wraps io.Reader and io.Seeker.
+type ReadSeeker interface {
+	io.Reader
+	io.Seeker
+}
+
+// ReadSeekCloser is implemented by afero.File. We use this as the common type for
+// content in Resource objects, even for strings.
+type ReadSeekCloser interface {
+	ReadSeeker
+	io.Closer
+}
+
+// ReadSeekerNoOpCloser implements ReadSeekCloser by doing nothing in Close.
+type ReadSeekerNoOpCloser struct {
+	ReadSeeker
+}
+
+// Close does nothing.
+func (r ReadSeekerNoOpCloser) Close() error {
+	return nil
+}
+
+// NewReadSeekerNoOpCloser creates a new ReadSeekerNoOpCloser with the given ReadSeeker.
+func NewReadSeekerNoOpCloser(r ReadSeeker) ReadSeekerNoOpCloser {
+	return ReadSeekerNoOpCloser{r}
+}
+
+// NewReadSeekerNoOpCloserFromString uses strings.NewReader to create a new ReadSeekerNoOpCloser
+// from the given string.
+func NewReadSeekerNoOpCloserFromString(content string) ReadSeekerNoOpCloser {
+	return ReadSeekerNoOpCloser{strings.NewReader(content)}
+}

common/hugio/writers.go

@@ -0,0 +1,76 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugio
+
+import (
+	"io"
+	"io/ioutil"
+)
+
+type multiWriteCloser struct {
+	io.Writer
+	closers []io.WriteCloser
+}
+
+func (m multiWriteCloser) Close() error {
+	var err error
+	for _, c := range m.closers {
+		if closeErr := c.Close(); err != nil {
+			err = closeErr
+		}
+	}
+	return err
+}
+
+// NewMultiWriteCloser creates a new io.WriteCloser that duplicates its writes to all the
+// provided writers.
+func NewMultiWriteCloser(writeClosers ...io.WriteCloser) io.WriteCloser {
+	writers := make([]io.Writer, len(writeClosers))
+	for i, w := range writeClosers {
+		writers[i] = w
+	}
+	return multiWriteCloser{Writer: io.MultiWriter(writers...), closers: writeClosers}
+}
+
+// ToWriteCloser creates an io.WriteCloser from the given io.Writer.
+// If it's not already, one will be created with a Close method that does nothing.
+func ToWriteCloser(w io.Writer) io.WriteCloser {
+	if rw, ok := w.(io.WriteCloser); ok {
+		return rw
+	}
+
+	return struct {
+		io.Writer
+		io.Closer
+	}{
+		w,
+		ioutil.NopCloser(nil),
+	}
+}
+
+// ToReadCloser creates an io.ReadCloser from the given io.Reader.
+// If it's not already, one will be created with a Close method that does nothing.
+func ToReadCloser(r io.Reader) io.ReadCloser {
+	if rc, ok := r.(io.ReadCloser); ok {
+		return rc
+	}
+
+	return struct {
+		io.Reader
+		io.Closer
+	}{
+		r,
+		ioutil.NopCloser(nil),
+	}
+}

common/loggers/loggers.go

@@ -0,0 +1,173 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package loggers
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"regexp"
+
+	"github.com/gohugoio/hugo/common/terminal"
+
+	jww "github.com/spf13/jwalterweatherman"
+)
+
+var (
+	// Counts ERROR logs to the global jww logger.
+	GlobalErrorCounter *jww.Counter
+)
+
+func init() {
+	GlobalErrorCounter = &jww.Counter{}
+	jww.SetLogListeners(jww.LogCounter(GlobalErrorCounter, jww.LevelError))
+}
+
+// Logger wraps a *loggers.Logger and some other related logging state.
+type Logger struct {
+	*jww.Notepad
+	ErrorCounter *jww.Counter
+
+	// This is only set in server mode.
+	errors *bytes.Buffer
+}
+
+func (l *Logger) Errors() string {
+	if l.errors == nil {
+		return ""
+	}
+	return ansiColorRe.ReplaceAllString(l.errors.String(), "")
+}
+
+// Reset resets the logger's internal state.
+func (l *Logger) Reset() {
+	l.ErrorCounter.Reset()
+	if l.errors != nil {
+		l.errors.Reset()
+	}
+}
+
+//  NewLogger creates a new Logger for the given thresholds
+func NewLogger(stdoutThreshold, logThreshold jww.Threshold, outHandle, logHandle io.Writer, saveErrors bool) *Logger {
+	return newLogger(stdoutThreshold, logThreshold, outHandle, logHandle, saveErrors)
+}
+
+// NewDebugLogger is a convenience function to create a debug logger.
+func NewDebugLogger() *Logger {
+	return newBasicLogger(jww.LevelDebug)
+}
+
+// NewWarningLogger is a convenience function to create a warning logger.
+func NewWarningLogger() *Logger {
+	return newBasicLogger(jww.LevelWarn)
+}
+
+// NewErrorLogger is a convenience function to create an error logger.
+func NewErrorLogger() *Logger {
+	return newBasicLogger(jww.LevelError)
+}
+
+var (
+	ansiColorRe = regexp.MustCompile("(?s)\\033\\[\\d*(;\\d*)*m")
+	errorRe     = regexp.MustCompile("^(ERROR|FATAL|WARN)")
+)
+
+type ansiCleaner struct {
+	w io.Writer
+}
+
+func (a ansiCleaner) Write(p []byte) (n int, err error) {
+	return a.w.Write(ansiColorRe.ReplaceAll(p, []byte("")))
+}
+
+type labelColorizer struct {
+	w io.Writer
+}
+
+func (a labelColorizer) Write(p []byte) (n int, err error) {
+	replaced := errorRe.ReplaceAllStringFunc(string(p), func(m string) string {
+		switch m {
+		case "ERROR", "FATAL":
+			return terminal.Error(m)
+		case "WARN":
+			return terminal.Warning(m)
+		default:
+			return m
+		}
+	})
+	// io.MultiWriter will abort if we return a bigger write count than input
+	// bytes, so we lie a little.
+	_, err = a.w.Write([]byte(replaced))
+	return len(p), err
+
+}
+
+// InitGlobalLogger initializes the global logger, used in some rare cases.
+func InitGlobalLogger(stdoutThreshold, logThreshold jww.Threshold, outHandle, logHandle io.Writer) {
+	outHandle, logHandle = getLogWriters(outHandle, logHandle)
+
+	jww.SetStdoutOutput(outHandle)
+	jww.SetLogOutput(logHandle)
+	jww.SetLogThreshold(logThreshold)
+	jww.SetStdoutThreshold(stdoutThreshold)
+
+}
+
+func getLogWriters(outHandle, logHandle io.Writer) (io.Writer, io.Writer) {
+	isTerm := terminal.IsTerminal(os.Stdout)
+	if logHandle != ioutil.Discard && isTerm {
+		// Remove any Ansi coloring from log output
+		logHandle = ansiCleaner{w: logHandle}
+	}
+
+	if isTerm {
+		outHandle = labelColorizer{w: outHandle}
+	}
+
+	return outHandle, logHandle
+
+}
+
+func newLogger(stdoutThreshold, logThreshold jww.Threshold, outHandle, logHandle io.Writer, saveErrors bool) *Logger {
+	errorCounter := &jww.Counter{}
+	outHandle, logHandle = getLogWriters(outHandle, logHandle)
+
+	listeners := []jww.LogListener{jww.LogCounter(errorCounter, jww.LevelError)}
+	var errorBuff *bytes.Buffer
+	if saveErrors {
+		errorBuff = new(bytes.Buffer)
+		errorCapture := func(t jww.Threshold) io.Writer {
+			if t != jww.LevelError {
+				// Only interested in ERROR
+				return nil
+			}
+
+			return errorBuff
+		}
+
+		listeners = append(listeners, errorCapture)
+	}
+
+	return &Logger{
+		Notepad:      jww.NewNotepad(stdoutThreshold, logThreshold, outHandle, logHandle, "", log.Ldate|log.Ltime, listeners...),
+		ErrorCounter: errorCounter,
+		errors:       errorBuff,
+	}
+}
+
+func newBasicLogger(t jww.Threshold) *Logger {
+	return newLogger(t, jww.LevelError, os.Stdout, ioutil.Discard, false)
+}

common/loggers/loggers_test.go

@@ -0,0 +1,32 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package loggers
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestLogger(t *testing.T) {
+	assert := require.New(t)
+	l := NewWarningLogger()
+
+	l.ERROR.Println("One error")
+	l.ERROR.Println("Two error")
+	l.WARN.Println("A warning")
+
+	assert.Equal(uint64(2), l.ErrorCounter.Count())
+
+}

common/maps/maps.go

@@ -0,0 +1,44 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package maps
+
+import (
+	"strings"
+
+	"github.com/spf13/cast"
+)
+
+// ToLower makes all the keys in the given map lower cased and will do so
+// recursively.
+// Notes:
+// * This will modify the map given.
+// * Any nested map[interface{}]interface{} will be converted to map[string]interface{}.
+func ToLower(m map[string]interface{}) {
+	for k, v := range m {
+		switch v.(type) {
+		case map[interface{}]interface{}:
+			v = cast.ToStringMap(v)
+			ToLower(v.(map[string]interface{}))
+		case map[string]interface{}:
+			ToLower(v.(map[string]interface{}))
+		}
+
+		lKey := strings.ToLower(k)
+		if k != lKey {
+			delete(m, k)
+			m[lKey] = v
+		}
+
+	}
+}

common/maps/maps_test.go

@@ -0,0 +1,72 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package maps
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestToLower(t *testing.T) {
+
+	tests := []struct {
+		input    map[string]interface{}
+		expected map[string]interface{}
+	}{
+		{
+			map[string]interface{}{
+				"abC": 32,
+			},
+			map[string]interface{}{
+				"abc": 32,
+			},
+		},
+		{
+			map[string]interface{}{
+				"abC": 32,
+				"deF": map[interface{}]interface{}{
+					23: "A value",
+					24: map[string]interface{}{
+						"AbCDe": "A value",
+						"eFgHi": "Another value",
+					},
+				},
+				"gHi": map[string]interface{}{
+					"J": 25,
+				},
+			},
+			map[string]interface{}{
+				"abc": 32,
+				"def": map[string]interface{}{
+					"23": "A value",
+					"24": map[string]interface{}{
+						"abcde": "A value",
+						"efghi": "Another value",
+					},
+				},
+				"ghi": map[string]interface{}{
+					"j": 25,
+				},
+			},
+		},
+	}
+
+	for i, test := range tests {
+		// ToLower modifies input.
+		ToLower(test.input)
+		if !reflect.DeepEqual(test.expected, test.input) {
+			t.Errorf("[%d] Expected\n%#v, got\n%#v\n", i, test.expected, test.input)
+		}
+	}
+}

common/maps/scratch.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,14 +11,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package hugolib
+package maps
 
 import (
 	"reflect"
 	"sort"
 	"sync"
 
-	"github.com/gohugoio/hugo/tpl/math"
+	"github.com/gohugoio/hugo/common/collections"
+	"github.com/gohugoio/hugo/common/math"
 )
 
 // Scratch is a writable context used for stateful operations in Page/Node rendering.
@@ -40,14 +41,12 @@ func (c *Scratch) Add(key string, newAddend interface{}) (string, error) {
 	if found {
 		var err error
 
-		addendV := reflect.ValueOf(existingAddend)
+		addendV := reflect.TypeOf(existingAddend)
 
 		if addendV.Kind() == reflect.Slice || addendV.Kind() == reflect.Array {
-			nav := reflect.ValueOf(newAddend)
-			if nav.Kind() == reflect.Slice || nav.Kind() == reflect.Array {
-				newVal = reflect.AppendSlice(addendV, nav).Interface()
-			} else {
-				newVal = reflect.Append(addendV, nav).Interface()
+			newVal, err = collections.Append(existingAddend, newAddend)
+			if err != nil {
+				return "", err
 			}
 		} else {
 			newVal, err = math.DoArithmetic(existingAddend, newAddend, '+')
@@ -73,7 +72,15 @@ func (c *Scratch) Set(key string, value interface{}) string {
 	return ""
 }
 
-// Get returns a value previously set by Add or Set
+// Delete deletes the given key.
+func (c *Scratch) Delete(key string) string {
+	c.mu.Lock()
+	delete(c.values, key)
+	c.mu.Unlock()
+	return ""
+}
+
+// Get returns a value previously set by Add or Set.
 func (c *Scratch) Get(key string) interface{} {
 	c.mu.RLock()
 	val := c.values[key]
@@ -96,7 +103,7 @@ func (c *Scratch) SetInMap(key string, mapKey string, value interface{}) string
 	return ""
 }
 
-// GetSortedMapValues returns a sorted map previously filled with SetInMap
+// GetSortedMapValues returns a sorted map previously filled with SetInMap.
 func (c *Scratch) GetSortedMapValues(key string) interface{} {
 	c.mu.RLock()
 
@@ -122,6 +129,7 @@ func (c *Scratch) GetSortedMapValues(key string) interface{} {
 	return sortedArray
 }
 
-func newScratch() *Scratch {
+// NewScratch returns a new instance Scratch.
+func NewScratch() *Scratch {
 	return &Scratch{values: make(map[string]interface{})}
 }

common/maps/scratch_test.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,36 +11,38 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package hugolib
+package maps
 
 import (
 	"reflect"
 	"sync"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestScratchAdd(t *testing.T) {
 	t.Parallel()
-	scratch := newScratch()
+	assert := require.New(t)
+
+	scratch := NewScratch()
 	scratch.Add("int1", 10)
 	scratch.Add("int1", 20)
 	scratch.Add("int2", 20)
 
-	assert.Equal(t, int64(30), scratch.Get("int1"))
-	assert.Equal(t, 20, scratch.Get("int2"))
+	assert.Equal(int64(30), scratch.Get("int1"))
+	assert.Equal(20, scratch.Get("int2"))
 
 	scratch.Add("float1", float64(10.5))
 	scratch.Add("float1", float64(20.1))
 
-	assert.Equal(t, float64(30.6), scratch.Get("float1"))
+	assert.Equal(float64(30.6), scratch.Get("float1"))
 
 	scratch.Add("string1", "Hello ")
 	scratch.Add("string1", "big ")
 	scratch.Add("string1", "World!")
 
-	assert.Equal(t, "Hello big World!", scratch.Get("string1"))
+	assert.Equal("Hello big World!", scratch.Get("string1"))
 
 	scratch.Add("scratch", scratch)
 	_, err := scratch.Add("scratch", scratch)
@@ -53,12 +55,14 @@ func TestScratchAdd(t *testing.T) {
 
 func TestScratchAddSlice(t *testing.T) {
 	t.Parallel()
-	scratch := newScratch()
+	assert := require.New(t)
+
+	scratch := NewScratch()
 
 	_, err := scratch.Add("intSlice", []int{1, 2})
-	assert.Nil(t, err)
+	assert.NoError(err)
 	_, err = scratch.Add("intSlice", 3)
-	assert.Nil(t, err)
+	assert.NoError(err)
 
 	sl := scratch.Get("intSlice")
 	expected := []int{1, 2, 3}
@@ -66,10 +70,9 @@ func TestScratchAddSlice(t *testing.T) {
 	if !reflect.DeepEqual(expected, sl) {
 		t.Errorf("Slice difference, go %q expected %q", sl, expected)
 	}
-
 	_, err = scratch.Add("intSlice", []int{4, 5})
 
-	assert.Nil(t, err)
+	assert.NoError(err)
 
 	sl = scratch.Get("intSlice")
 	expected = []int{1, 2, 3, 4, 5}
@@ -77,20 +80,61 @@ func TestScratchAddSlice(t *testing.T) {
 	if !reflect.DeepEqual(expected, sl) {
 		t.Errorf("Slice difference, go %q expected %q", sl, expected)
 	}
+}
+
+// https://github.com/gohugoio/hugo/issues/5275
+func TestScratchAddTypedSliceToInterfaceSlice(t *testing.T) {
+	t.Parallel()
+	assert := require.New(t)
+
+	scratch := NewScratch()
+	scratch.Set("slice", []interface{}{})
+
+	_, err := scratch.Add("slice", []int{1, 2})
+	assert.NoError(err)
+	assert.Equal([]int{1, 2}, scratch.Get("slice"))
+
+}
+
+// https://github.com/gohugoio/hugo/issues/5361
+func TestScratchAddDifferentTypedSliceToInterfaceSlice(t *testing.T) {
+	t.Parallel()
+	assert := require.New(t)
+
+	scratch := NewScratch()
+	scratch.Set("slice", []string{"foo"})
+
+	_, err := scratch.Add("slice", []int{1, 2})
+	assert.NoError(err)
+	assert.Equal([]interface{}{"foo", 1, 2}, scratch.Get("slice"))
 
 }
 
 func TestScratchSet(t *testing.T) {
 	t.Parallel()
-	scratch := newScratch()
+	assert := require.New(t)
+
+	scratch := NewScratch()
 	scratch.Set("key", "val")
-	assert.Equal(t, "val", scratch.Get("key"))
+	assert.Equal("val", scratch.Get("key"))
+}
+
+func TestScratchDelete(t *testing.T) {
+	t.Parallel()
+	assert := require.New(t)
+
+	scratch := NewScratch()
+	scratch.Set("key", "val")
+	scratch.Delete("key")
+	scratch.Add("key", "Lucy Parsons")
+	assert.Equal("Lucy Parsons", scratch.Get("key"))
 }
 
 // Issue #2005
 func TestScratchInParallel(t *testing.T) {
 	var wg sync.WaitGroup
-	scratch := newScratch()
+	scratch := NewScratch()
+
 	key := "counter"
 	scratch.Set(key, int64(1))
 	for i := 1; i <= 10; i++ {
@@ -124,7 +168,7 @@ func TestScratchInParallel(t *testing.T) {
 
 func TestScratchGet(t *testing.T) {
 	t.Parallel()
-	scratch := newScratch()
+	scratch := NewScratch()
 	nothing := scratch.Get("nothing")
 	if nothing != nil {
 		t.Errorf("Should not return anything, but got %v", nothing)
@@ -133,18 +177,20 @@ func TestScratchGet(t *testing.T) {
 
 func TestScratchSetInMap(t *testing.T) {
 	t.Parallel()
-	scratch := newScratch()
+	assert := require.New(t)
+
+	scratch := NewScratch()
 	scratch.SetInMap("key", "lux", "Lux")
 	scratch.SetInMap("key", "abc", "Abc")
 	scratch.SetInMap("key", "zyx", "Zyx")
 	scratch.SetInMap("key", "abc", "Abc (updated)")
 	scratch.SetInMap("key", "def", "Def")
-	assert.Equal(t, []interface{}{0: "Abc (updated)", 1: "Def", 2: "Lux", 3: "Zyx"}, scratch.GetSortedMapValues("key"))
+	assert.Equal([]interface{}{0: "Abc (updated)", 1: "Def", 2: "Lux", 3: "Zyx"}, scratch.GetSortedMapValues("key"))
 }
 
 func TestScratchGetSortedMapValues(t *testing.T) {
 	t.Parallel()
-	scratch := newScratch()
+	scratch := NewScratch()
 	nothing := scratch.GetSortedMapValues("nothing")
 	if nothing != nil {
 		t.Errorf("Should not return anything, but got %v", nothing)
@@ -152,7 +198,7 @@ func TestScratchGetSortedMapValues(t *testing.T) {
 }
 
 func BenchmarkScratchGet(b *testing.B) {
-	scratch := newScratch()
+	scratch := NewScratch()
 	scratch.Add("A", 1)
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {

common/math/math.go

@@ -0,0 +1,135 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package math
+
+import (
+	"errors"
+	"reflect"
+)
+
+// DoArithmetic performs arithmetic operations (+,-,*,/) using reflection to
+// determine the type of the two terms.
+func DoArithmetic(a, b interface{}, op rune) (interface{}, error) {
+	av := reflect.ValueOf(a)
+	bv := reflect.ValueOf(b)
+	var ai, bi int64
+	var af, bf float64
+	var au, bu uint64
+	switch av.Kind() {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		ai = av.Int()
+		switch bv.Kind() {
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			bi = bv.Int()
+		case reflect.Float32, reflect.Float64:
+			af = float64(ai) // may overflow
+			ai = 0
+			bf = bv.Float()
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			bu = bv.Uint()
+			if ai >= 0 {
+				au = uint64(ai)
+				ai = 0
+			} else {
+				bi = int64(bu) // may overflow
+				bu = 0
+			}
+		default:
+			return nil, errors.New("Can't apply the operator to the values")
+		}
+	case reflect.Float32, reflect.Float64:
+		af = av.Float()
+		switch bv.Kind() {
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			bf = float64(bv.Int()) // may overflow
+		case reflect.Float32, reflect.Float64:
+			bf = bv.Float()
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			bf = float64(bv.Uint()) // may overflow
+		default:
+			return nil, errors.New("Can't apply the operator to the values")
+		}
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		au = av.Uint()
+		switch bv.Kind() {
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			bi = bv.Int()
+			if bi >= 0 {
+				bu = uint64(bi)
+				bi = 0
+			} else {
+				ai = int64(au) // may overflow
+				au = 0
+			}
+		case reflect.Float32, reflect.Float64:
+			af = float64(au) // may overflow
+			au = 0
+			bf = bv.Float()
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			bu = bv.Uint()
+		default:
+			return nil, errors.New("Can't apply the operator to the values")
+		}
+	case reflect.String:
+		as := av.String()
+		if bv.Kind() == reflect.String && op == '+' {
+			bs := bv.String()
+			return as + bs, nil
+		}
+		return nil, errors.New("Can't apply the operator to the values")
+	default:
+		return nil, errors.New("Can't apply the operator to the values")
+	}
+
+	switch op {
+	case '+':
+		if ai != 0 || bi != 0 {
+			return ai + bi, nil
+		} else if af != 0 || bf != 0 {
+			return af + bf, nil
+		} else if au != 0 || bu != 0 {
+			return au + bu, nil
+		}
+		return 0, nil
+	case '-':
+		if ai != 0 || bi != 0 {
+			return ai - bi, nil
+		} else if af != 0 || bf != 0 {
+			return af - bf, nil
+		} else if au != 0 || bu != 0 {
+			return au - bu, nil
+		}
+		return 0, nil
+	case '*':
+		if ai != 0 || bi != 0 {
+			return ai * bi, nil
+		} else if af != 0 || bf != 0 {
+			return af * bf, nil
+		} else if au != 0 || bu != 0 {
+			return au * bu, nil
+		}
+		return 0, nil
+	case '/':
+		if bi != 0 {
+			return ai / bi, nil
+		} else if bf != 0 {
+			return af / bf, nil
+		} else if bu != 0 {
+			return au / bu, nil
+		}
+		return nil, errors.New("Can't divide the value by 0")
+	default:
+		return nil, errors.New("There is no such an operation")
+	}
+}

common/math/math_test.go

@@ -0,0 +1,109 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package math
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/alecthomas/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDoArithmetic(t *testing.T) {
+	t.Parallel()
+
+	for i, test := range []struct {
+		a      interface{}
+		b      interface{}
+		op     rune
+		expect interface{}
+	}{
+		{3, 2, '+', int64(5)},
+		{3, 2, '-', int64(1)},
+		{3, 2, '*', int64(6)},
+		{3, 2, '/', int64(1)},
+		{3.0, 2, '+', float64(5)},
+		{3.0, 2, '-', float64(1)},
+		{3.0, 2, '*', float64(6)},
+		{3.0, 2, '/', float64(1.5)},
+		{3, 2.0, '+', float64(5)},
+		{3, 2.0, '-', float64(1)},
+		{3, 2.0, '*', float64(6)},
+		{3, 2.0, '/', float64(1.5)},
+		{3.0, 2.0, '+', float64(5)},
+		{3.0, 2.0, '-', float64(1)},
+		{3.0, 2.0, '*', float64(6)},
+		{3.0, 2.0, '/', float64(1.5)},
+		{uint(3), uint(2), '+', uint64(5)},
+		{uint(3), uint(2), '-', uint64(1)},
+		{uint(3), uint(2), '*', uint64(6)},
+		{uint(3), uint(2), '/', uint64(1)},
+		{uint(3), 2, '+', uint64(5)},
+		{uint(3), 2, '-', uint64(1)},
+		{uint(3), 2, '*', uint64(6)},
+		{uint(3), 2, '/', uint64(1)},
+		{3, uint(2), '+', uint64(5)},
+		{3, uint(2), '-', uint64(1)},
+		{3, uint(2), '*', uint64(6)},
+		{3, uint(2), '/', uint64(1)},
+		{uint(3), -2, '+', int64(1)},
+		{uint(3), -2, '-', int64(5)},
+		{uint(3), -2, '*', int64(-6)},
+		{uint(3), -2, '/', int64(-1)},
+		{-3, uint(2), '+', int64(-1)},
+		{-3, uint(2), '-', int64(-5)},
+		{-3, uint(2), '*', int64(-6)},
+		{-3, uint(2), '/', int64(-1)},
+		{uint(3), 2.0, '+', float64(5)},
+		{uint(3), 2.0, '-', float64(1)},
+		{uint(3), 2.0, '*', float64(6)},
+		{uint(3), 2.0, '/', float64(1.5)},
+		{3.0, uint(2), '+', float64(5)},
+		{3.0, uint(2), '-', float64(1)},
+		{3.0, uint(2), '*', float64(6)},
+		{3.0, uint(2), '/', float64(1.5)},
+		{0, 0, '+', 0},
+		{0, 0, '-', 0},
+		{0, 0, '*', 0},
+		{"foo", "bar", '+', "foobar"},
+		{3, 0, '/', false},
+		{3.0, 0, '/', false},
+		{3, 0.0, '/', false},
+		{uint(3), uint(0), '/', false},
+		{3, uint(0), '/', false},
+		{-3, uint(0), '/', false},
+		{uint(3), 0, '/', false},
+		{3.0, uint(0), '/', false},
+		{uint(3), 0.0, '/', false},
+		{3, "foo", '+', false},
+		{3.0, "foo", '+', false},
+		{uint(3), "foo", '+', false},
+		{"foo", 3, '+', false},
+		{"foo", "bar", '-', false},
+		{3, 2, '%', false},
+	} {
+		errMsg := fmt.Sprintf("[%d] %v", i, test)
+
+		result, err := DoArithmetic(test.a, test.b, test.op)
+
+		if b, ok := test.expect.(bool); ok && !b {
+			require.Error(t, err, errMsg)
+			continue
+		}
+
+		require.NoError(t, err, errMsg)
+		assert.Equal(t, test.expect, result, errMsg)
+	}
+}

common/terminal/colors.go

@@ -0,0 +1,70 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package terminal contains helper for the terminal, such as coloring output.
+package terminal
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"strings"
+
+	isatty "github.com/mattn/go-isatty"
+)
+
+const (
+	errorColor   = "\033[1;31m%s\033[0m"
+	warningColor = "\033[0;33m%s\033[0m"
+	noticeColor  = "\033[1;36m%s\033[0m"
+)
+
+// IsTerminal return true if the file descriptor is terminal and the TERM
+// environment variable isn't a dumb one.
+func IsTerminal(f *os.File) bool {
+	if runtime.GOOS == "windows" {
+		return false
+	}
+
+	fd := f.Fd()
+	return os.Getenv("TERM") != "dumb" && (isatty.IsTerminal(fd) || isatty.IsCygwinTerminal(fd))
+}
+
+// Notice colorizes the string in a noticeable color.
+func Notice(s string) string {
+	return colorize(s, noticeColor)
+}
+
+// Error colorizes the string in a colour that grabs attention.
+func Error(s string) string {
+	return colorize(s, errorColor)
+}
+
+// Warning colorizes the string in a colour that warns.
+func Warning(s string) string {
+	return colorize(s, warningColor)
+}
+
+// colorize s in color.
+func colorize(s, color string) string {
+	s = fmt.Sprintf(color, doublePercent(s))
+	return singlePercent(s)
+}
+
+func doublePercent(str string) string {
+	return strings.Replace(str, "%", "%%", -1)
+}
+
+func singlePercent(str string) string {
+	return strings.Replace(str, "%%", "%", -1)
+}

common/text/position.go

@@ -0,0 +1,99 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package text
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/gohugoio/hugo/common/terminal"
+)
+
+// Positioner represents a thing that knows its position in a text file or stream,
+// typically an error.
+type Positioner interface {
+	Position() Position
+}
+
+// Position holds a source position in a text file or stream.
+type Position struct {
+	Filename     string // filename, if any
+	Offset       int    // byte offset, starting at 0. It's set to -1 if not provided.
+	LineNumber   int    // line number, starting at 1
+	ColumnNumber int    // column number, starting at 1 (character count per line)
+}
+
+func (pos Position) String() string {
+	if pos.Filename == "" {
+		pos.Filename = "<stream>"
+	}
+	return positionStringFormatfunc(pos)
+}
+
+// IsValid returns true if line number is > 0.
+func (pos Position) IsValid() bool {
+	return pos.LineNumber > 0
+}
+
+var positionStringFormatfunc func(p Position) string
+
+func createPositionStringFormatter(formatStr string) func(p Position) string {
+
+	if formatStr == "" {
+		formatStr = "\":file::line::col\""
+	}
+
+	var identifiers = []string{":file", ":line", ":col"}
+	var identifiersFound []string
+
+	for i := range formatStr {
+		for _, id := range identifiers {
+			if strings.HasPrefix(formatStr[i:], id) {
+				identifiersFound = append(identifiersFound, id)
+			}
+		}
+	}
+
+	replacer := strings.NewReplacer(":file", "%s", ":line", "%d", ":col", "%d")
+	format := replacer.Replace(formatStr)
+
+	f := func(pos Position) string {
+		args := make([]interface{}, len(identifiersFound))
+		for i, id := range identifiersFound {
+			switch id {
+			case ":file":
+				args[i] = pos.Filename
+			case ":line":
+				args[i] = pos.LineNumber
+			case ":col":
+				args[i] = pos.ColumnNumber
+			}
+		}
+
+		msg := fmt.Sprintf(format, args...)
+
+		if terminal.IsTerminal(os.Stdout) {
+			return terminal.Notice(msg)
+		}
+
+		return msg
+	}
+
+	return f
+}
+
+func init() {
+	positionStringFormatfunc = createPositionStringFormatter(os.Getenv("HUGO_FILE_LOG_FORMAT"))
+}

common/text/position_test.go

@@ -0,0 +1,33 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package text
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestPositionStringFormatter(t *testing.T) {
+	assert := require.New(t)
+
+	pos := Position{Filename: "/my/file.txt", LineNumber: 12, ColumnNumber: 13, Offset: 14}
+
+	assert.Equal("/my/file.txt|13|12", createPositionStringFormatter(":file|:col|:line")(pos))
+	assert.Equal("13|/my/file.txt|12", createPositionStringFormatter(":col|:file|:line")(pos))
+	assert.Equal("好:13", createPositionStringFormatter("好::col")(pos))
+	assert.Equal("\"/my/file.txt:12:13\"", createPositionStringFormatter("")(pos))
+	assert.Equal("\"/my/file.txt:12:13\"", pos.String())
+
+}

common/types/evictingqueue.go

@@ -52,6 +52,13 @@ func (q *EvictingStringQueue) Add(v string) {
 	q.mu.Unlock()
 }
 
+// Contains returns whether the queue contains v.
+func (q *EvictingStringQueue) Contains(v string) bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return q.set[v]
+}
+
 // Peek looks at the last element added to the queue.
 func (q *EvictingStringQueue) Peek() string {
 	q.mu.Lock()

common/types/evictingqueue_test.go

@@ -36,6 +36,9 @@ func TestEvictingStringQueue(t *testing.T) {
 	queue.Add("a")
 	queue.Add("b")
 
+	assert.True(queue.Contains("a"))
+	assert.False(queue.Contains("foo"))
+
 	assert.Equal([]string{"b", "a"}, queue.PeekAll())
 	assert.Equal("b", queue.Peek())
 	queue.Add("c")

common/types/types.go

@@ -1,4 +1,4 @@
-// Copyright 2017-present The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,6 +20,12 @@ import (
 	"github.com/spf13/cast"
 )
 
+// KeyValueStr is a string tuple.
+type KeyValueStr struct {
+	Key   string
+	Value string
+}
+
 // KeyValues holds an key and a slice of values.
 type KeyValues struct {
 	Key    interface{}
@@ -35,6 +41,8 @@ func (k KeyValues) String() string {
 	return fmt.Sprintf("%v: %v", k.Key, k.Values)
 }
 
+// NewKeyValuesStrings takes a given key and slice of values and returns a new
+// KeyValues struct.
 func NewKeyValuesStrings(key string, values ...string) KeyValues {
 	iv := make([]interface{}, len(values))
 	for i := 0; i < len(values); i++ {

common/urls/ref.go

@@ -0,0 +1,22 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package urls
+
+// RefLinker is implemented by those who support reference linking.
+// args must contain a path, but can also point to the target
+// language or output format.
+type RefLinker interface {
+	Ref(args map[string]interface{}) (string, error)
+	RelRef(args map[string]interface{}) (string, error)
+}

compare/compare.go

@@ -19,3 +19,11 @@ package compare
 type Eqer interface {
 	Eq(other interface{}) bool
 }
+
+// Comparer can be used to compare two values.
+// This will be used when using the le, ge etc. operators in the templates.
+// Compare returns -1 if the given version is less than, 0 if equal and 1 if greater than
+// the running version.
+type Comparer interface {
+	Compare(other interface{}) int
+}

config/configProvider.go

@@ -13,6 +13,14 @@
 
 package config
 
+import (
+	"strings"
+
+	"github.com/spf13/cast"
+
+	"github.com/spf13/viper"
+)
+
 // Provider provides the configuration settings for Hugo.
 type Provider interface {
 	GetString(key string) string
@@ -20,7 +28,29 @@ type Provider interface {
 	GetBool(key string) bool
 	GetStringMap(key string) map[string]interface{}
 	GetStringMapString(key string) map[string]string
+	GetStringSlice(key string) []string
 	Get(key string) interface{}
 	Set(key string, value interface{})
 	IsSet(key string) bool
 }
+
+// FromConfigString creates a config from the given YAML, JSON or TOML config. This is useful in tests.
+func FromConfigString(config, configType string) (Provider, error) {
+	v := viper.New()
+	v.SetConfigType(configType)
+	if err := v.ReadConfig(strings.NewReader(config)); err != nil {
+		return nil, err
+	}
+	return v, nil
+}
+
+// GetStringSlicePreserveString returns a string slice from the given config and key.
+// It differs from the GetStringSlice method in that if the config value is a string,
+// we do not attempt to split it into fields.
+func GetStringSlicePreserveString(cfg Provider, key string) []string {
+	sd := cfg.Get(key)
+	if sds, ok := sd.(string); ok {
+		return []string{sds}
+	}
+	return cast.ToStringSlice(sd)
+}

config/configProvider_test.go

@@ -0,0 +1,36 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetStringSlicePreserveString(t *testing.T) {
+	assert := require.New(t)
+	cfg := viper.New()
+
+	s := "This is a string"
+	sSlice := []string{"This", "is", "a", "slice"}
+
+	cfg.Set("s1", s)
+	cfg.Set("s2", sSlice)
+
+	assert.Equal([]string{s}, GetStringSlicePreserveString(cfg, "s1"))
+	assert.Equal(sSlice, GetStringSlicePreserveString(cfg, "s2"))
+	assert.Nil(GetStringSlicePreserveString(cfg, "s3"))
+}

config/privacy/privacyConfig.go

@@ -0,0 +1,110 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package privacy
+
+import (
+	"github.com/gohugoio/hugo/config"
+	"github.com/mitchellh/mapstructure"
+)
+
+const privacyConfigKey = "privacy"
+
+// Service is the common values for a service in a policy definition.
+type Service struct {
+	Disable bool
+}
+
+// Config is a privacy configuration for all the relevant services in Hugo.
+type Config struct {
+	Disqus          Disqus
+	GoogleAnalytics GoogleAnalytics
+	Instagram       Instagram
+	Twitter         Twitter
+	Vimeo           Vimeo
+	YouTube         YouTube
+}
+
+// Disqus holds the privacy configuration settings related to the Disqus template.
+type Disqus struct {
+	Service `mapstructure:",squash"`
+}
+
+// GoogleAnalytics holds the privacy configuration settings related to the Google Analytics template.
+type GoogleAnalytics struct {
+	Service `mapstructure:",squash"`
+
+	// Enabling this will disable the use of Cookies and use Session Storage to Store the GA Client ID.
+	UseSessionStorage bool
+
+	// Enabling this will make the GA templates respect the
+	// "Do Not Track" HTTP header. See  https://www.paulfurley.com/google-analytics-dnt/.
+	RespectDoNotTrack bool
+
+	// Enabling this will make it so the users' IP addresses are anonymized within Google Analytics.
+	AnonymizeIP bool
+}
+
+// Instagram holds the privacy configuration settings related to the Instagram shortcode.
+type Instagram struct {
+	Service `mapstructure:",squash"`
+
+	// If simple mode is enabled, a static and no-JS version of the Instagram
+	// image card will be built.
+	Simple bool
+}
+
+// Twitter holds the privacy configuration settingsrelated to the Twitter shortcode.
+type Twitter struct {
+	Service `mapstructure:",squash"`
+
+	// When set to true, the Tweet and its embedded page on your site are not used
+	// for purposes that include personalized suggestions and personalized ads.
+	EnableDNT bool
+
+	// If simple mode is enabled, a static and no-JS version of the Tweet will be built.
+	Simple bool
+}
+
+// Vimeo holds the privacy configuration settingsrelated to the Vimeo shortcode.
+type Vimeo struct {
+	Service `mapstructure:",squash"`
+
+	// If simple mode is enabled, only a thumbnail is fetched from i.vimeocdn.com and
+	// shown with a play button overlaid. If a user clicks the button, he/she will
+	// be taken to the video page on vimeo.com in a new browser tab.
+	Simple bool
+}
+
+// YouTube holds the privacy configuration settingsrelated to the YouTube shortcode.
+type YouTube struct {
+	Service `mapstructure:",squash"`
+
+	// When you turn on privacy-enhanced mode,
+	// YouTube won’t store information about visitors on your website
+	// unless the user plays the embedded video.
+	PrivacyEnhanced bool
+}
+
+// DecodeConfig creates a privacy Config from a given Hugo configuration.
+func DecodeConfig(cfg config.Provider) (pc Config, err error) {
+	if !cfg.IsSet(privacyConfigKey) {
+		return
+	}
+
+	m := cfg.GetStringMap(privacyConfigKey)
+
+	err = mapstructure.WeakDecode(m, &pc)
+
+	return
+}

config/privacy/privacyConfig_test.go

@@ -0,0 +1,104 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package privacy
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/config"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDecodeConfigFromTOML(t *testing.T) {
+	assert := require.New(t)
+
+	tomlConfig := `
+
+someOtherValue = "foo"
+
+[privacy]
+[privacy.disqus]
+disable = true
+[privacy.googleAnalytics]
+disable = true
+respectDoNotTrack = true
+anonymizeIP = true
+useSessionStorage = true
+[privacy.instagram]
+disable = true
+simple = true
+[privacy.twitter]
+disable = true
+enableDNT = true
+simple = true
+[privacy.vimeo]
+disable = true
+simple = true
+[privacy.youtube]
+disable = true
+privacyEnhanced = true
+simple = true
+`
+	cfg, err := config.FromConfigString(tomlConfig, "toml")
+	assert.NoError(err)
+
+	pc, err := DecodeConfig(cfg)
+	assert.NoError(err)
+	assert.NotNil(pc)
+
+	assert.True(pc.Disqus.Disable)
+	assert.True(pc.GoogleAnalytics.Disable)
+	assert.True(pc.GoogleAnalytics.RespectDoNotTrack)
+	assert.True(pc.GoogleAnalytics.AnonymizeIP)
+	assert.True(pc.GoogleAnalytics.UseSessionStorage)
+	assert.True(pc.Instagram.Disable)
+	assert.True(pc.Instagram.Simple)
+	assert.True(pc.Twitter.Disable)
+	assert.True(pc.Twitter.EnableDNT)
+	assert.True(pc.Twitter.Simple)
+	assert.True(pc.Vimeo.Disable)
+	assert.True(pc.Vimeo.Simple)
+	assert.True(pc.YouTube.PrivacyEnhanced)
+	assert.True(pc.YouTube.Disable)
+}
+
+func TestDecodeConfigFromTOMLCaseInsensitive(t *testing.T) {
+	assert := require.New(t)
+
+	tomlConfig := `
+
+someOtherValue = "foo"
+
+[Privacy]
+[Privacy.YouTube]
+PrivacyENhanced = true
+`
+	cfg, err := config.FromConfigString(tomlConfig, "toml")
+	assert.NoError(err)
+
+	pc, err := DecodeConfig(cfg)
+	assert.NoError(err)
+	assert.NotNil(pc)
+	assert.True(pc.YouTube.PrivacyEnhanced)
+}
+
+func TestDecodeConfigDefault(t *testing.T) {
+	assert := require.New(t)
+
+	pc, err := DecodeConfig(viper.New())
+	assert.NoError(err)
+	assert.NotNil(pc)
+	assert.False(pc.YouTube.PrivacyEnhanced)
+}

config/services/servicesConfig.go

@@ -0,0 +1,80 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package services
+
+import (
+	"github.com/gohugoio/hugo/config"
+	"github.com/mitchellh/mapstructure"
+)
+
+const (
+	servicesConfigKey = "services"
+
+	disqusShortnameKey = "disqusshortname"
+	googleAnalyticsKey = "googleanalytics"
+)
+
+// Config is a privacy configuration for all the relevant services in Hugo.
+type Config struct {
+	Disqus          Disqus
+	GoogleAnalytics GoogleAnalytics
+	Instagram       Instagram
+	Twitter         Twitter
+}
+
+// Disqus holds the functional configuration settings related to the Disqus template.
+type Disqus struct {
+	// A Shortname is the unique identifier assigned to a Disqus site.
+	Shortname string
+}
+
+// GoogleAnalytics holds the functional configuration settings related to the Google Analytics template.
+type GoogleAnalytics struct {
+	// The GA tracking ID.
+	ID string
+}
+
+// Instagram holds the functional configuration settings related to the Instagram shortcodes.
+type Instagram struct {
+	// The Simple variant of the Instagram is decorated with Bootstrap 4 card classes.
+	// This means that if you use Bootstrap 4 or want to provide your own CSS, you want
+	// to disable the inline CSS provided by Hugo.
+	DisableInlineCSS bool
+}
+
+// Twitter holds the functional configuration settings related to the Twitter shortcodes.
+type Twitter struct {
+	// The Simple variant of Twitter is decorated with a basic set of inline styles.
+	// This means that if you want to provide your own CSS, you want
+	// to disable the inline CSS provided by Hugo.
+	DisableInlineCSS bool
+}
+
+// DecodeConfig creates a services Config from a given Hugo configuration.
+func DecodeConfig(cfg config.Provider) (c Config, err error) {
+	m := cfg.GetStringMap(servicesConfigKey)
+
+	err = mapstructure.WeakDecode(m, &c)
+
+	// Keep backwards compatibility.
+	if c.GoogleAnalytics.ID == "" {
+		// Try the global config
+		c.GoogleAnalytics.ID = cfg.GetString(googleAnalyticsKey)
+	}
+	if c.Disqus.Shortname == "" {
+		c.Disqus.Shortname = cfg.GetString(disqusShortnameKey)
+	}
+
+	return
+}

config/services/servicesConfig_test.go

@@ -0,0 +1,69 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package services
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/config"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDecodeConfigFromTOML(t *testing.T) {
+	assert := require.New(t)
+
+	tomlConfig := `
+
+someOtherValue = "foo"
+
+[services]
+[services.disqus]
+shortname = "DS"
+[services.googleAnalytics]
+id = "ga_id"
+[services.instagram]
+disableInlineCSS = true
+[services.twitter]
+disableInlineCSS = true
+`
+	cfg, err := config.FromConfigString(tomlConfig, "toml")
+	assert.NoError(err)
+
+	config, err := DecodeConfig(cfg)
+	assert.NoError(err)
+	assert.NotNil(config)
+
+	assert.Equal("DS", config.Disqus.Shortname)
+	assert.Equal("ga_id", config.GoogleAnalytics.ID)
+
+	assert.True(config.Instagram.DisableInlineCSS)
+}
+
+// Support old root-level GA settings etc.
+func TestUseSettingsFromRootIfSet(t *testing.T) {
+	assert := require.New(t)
+
+	cfg := viper.New()
+	cfg.Set("disqusShortname", "root_short")
+	cfg.Set("googleAnalytics", "ga_root")
+
+	config, err := DecodeConfig(cfg)
+	assert.NoError(err)
+	assert.NotNil(config)
+
+	assert.Equal("root_short", config.Disqus.Shortname)
+	assert.Equal("ga_root", config.GoogleAnalytics.ID)
+
+}

create/content.go

@@ -16,55 +16,80 @@ package create
 
 import (
 	"bytes"
+
+	"github.com/pkg/errors"
+
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
+
+	"github.com/gohugoio/hugo/hugofs"
 
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugolib"
+	"github.com/spf13/afero"
 	jww "github.com/spf13/jwalterweatherman"
 )
 
 // NewContent creates a new content file in the content directory based upon the
 // given kind, which is used to lookup an archetype.
 func NewContent(
-	ps *helpers.PathSpec,
-	siteFactory func(filename string, siteUsed bool) (*hugolib.Site, error), kind, targetPath string) error {
+	sites *hugolib.HugoSites, kind, targetPath string) error {
+	targetPath = filepath.Clean(targetPath)
 	ext := helpers.Ext(targetPath)
+	ps := sites.PathSpec
+	archetypeFs := ps.BaseFs.SourceFilesystems.Archetypes.Fs
+	sourceFs := ps.Fs.Source
 
 	jww.INFO.Printf("attempting to create %q of %q of ext %q", targetPath, kind, ext)
 
-	archetypeFilename := findArchetype(ps, kind, ext)
+	archetypeFilename, isDir := findArchetype(ps, kind, ext)
+	contentPath, s := resolveContentPath(sites, sourceFs, targetPath)
+
+	if isDir {
+
+		langFs := hugofs.NewLanguageFs(s.Language.Lang, sites.LanguageSet(), archetypeFs)
+
+		cm, err := mapArcheTypeDir(ps, langFs, archetypeFilename)
+		if err != nil {
+			return err
+		}
+
+		if cm.siteUsed {
+			if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
+				return err
+			}
+		}
+
+		name := filepath.Base(targetPath)
+		return newContentFromDir(archetypeFilename, sites, archetypeFs, sourceFs, cm, name, contentPath)
+	}
 
 	// Building the sites can be expensive, so only do it if really needed.
 	siteUsed := false
 
 	if archetypeFilename != "" {
-		f, err := ps.Fs.Source.Open(archetypeFilename)
+
+		var err error
+		siteUsed, err = usesSiteVar(archetypeFs, archetypeFilename)
 		if err != nil {
 			return err
 		}
-		defer f.Close()
+	}
 
-		if helpers.ReaderContains(f, []byte(".Site")) {
-			siteUsed = true
+	if siteUsed {
+		if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
+			return err
 		}
 	}
 
-	s, err := siteFactory(targetPath, siteUsed)
+	content, err := executeArcheTypeAsTemplate(s, "", kind, targetPath, archetypeFilename)
 	if err != nil {
 		return err
 	}
 
-	var content []byte
-
-	content, err = executeArcheTypeAsTemplate(s, kind, targetPath, archetypeFilename)
-	if err != nil {
-		return err
-	}
-
-	contentPath := s.PathSpec.AbsPathify(filepath.Join(s.Cfg.GetString("contentDir"), targetPath))
-
 	if err := helpers.SafeWriteToDisk(contentPath, bytes.NewReader(content), s.Fs.Source); err != nil {
 		return err
 	}
@@ -86,44 +111,200 @@ func NewContent(
 	return nil
 }
 
-// FindArchetype takes a given kind/archetype of content and returns an output
-// path for that archetype.  If no archetype is found, an empty string is
-// returned.
-func findArchetype(ps *helpers.PathSpec, kind, ext string) (outpath string) {
-	search := []string{ps.AbsPathify(ps.Cfg.GetString("archetypeDir"))}
-
-	if ps.Cfg.GetString("theme") != "" {
-		themeDir := filepath.Join(ps.AbsPathify(ps.Cfg.GetString("themesDir")+"/"+ps.Cfg.GetString("theme")), "/archetypes/")
-		if _, err := ps.Fs.Source.Stat(themeDir); os.IsNotExist(err) {
-			jww.ERROR.Printf("Unable to find archetypes directory for theme %q at %q", ps.Cfg.GetString("theme"), themeDir)
-		} else {
-			search = append(search, themeDir)
+func targetSite(sites *hugolib.HugoSites, fi *hugofs.LanguageFileInfo) *hugolib.Site {
+	for _, s := range sites.Sites {
+		if fi.Lang() == s.Language.Lang {
+			return s
 		}
 	}
-
-	for _, x := range search {
-		// If the new content isn't in a subdirectory, kind == "".
-		// Therefore it should be excluded otherwise `is a directory`
-		// error will occur. github.com/gohugoio/hugo/issues/411
-		var pathsToCheck = []string{"default"}
-
-		if ext != "" {
-			if kind != "" {
-				pathsToCheck = append([]string{kind + ext, "default" + ext}, pathsToCheck...)
-			} else {
-				pathsToCheck = append([]string{"default" + ext}, pathsToCheck...)
-			}
-		}
-
-		for _, p := range pathsToCheck {
-			curpath := filepath.Join(x, p)
-			jww.DEBUG.Println("checking", curpath, "for archetypes")
-			if exists, _ := helpers.Exists(curpath, ps.Fs.Source); exists {
-				jww.INFO.Println("curpath: " + curpath)
-				return curpath
-			}
-		}
-	}
-
-	return ""
+	return sites.Sites[0]
+}
+
+func newContentFromDir(
+	archetypeDir string,
+	sites *hugolib.HugoSites,
+	sourceFs, targetFs afero.Fs,
+	cm archetypeMap, name, targetPath string) error {
+
+	for _, f := range cm.otherFiles {
+		filename := f.Filename()
+		// Just copy the file to destination.
+		in, err := sourceFs.Open(filename)
+		if err != nil {
+			return errors.Wrap(err, "failed to open non-content file")
+		}
+
+		targetFilename := filepath.Join(targetPath, strings.TrimPrefix(filename, archetypeDir))
+
+		targetDir := filepath.Dir(targetFilename)
+		if err := targetFs.MkdirAll(targetDir, 0777); err != nil && !os.IsExist(err) {
+			return errors.Wrapf(err, "failed to create target directory for %s:", targetDir)
+		}
+
+		out, err := targetFs.Create(targetFilename)
+
+		_, err = io.Copy(out, in)
+		if err != nil {
+			return err
+		}
+
+		in.Close()
+		out.Close()
+	}
+
+	for _, f := range cm.contentFiles {
+		filename := f.Filename()
+		s := targetSite(sites, f)
+		targetFilename := filepath.Join(targetPath, strings.TrimPrefix(filename, archetypeDir))
+
+		content, err := executeArcheTypeAsTemplate(s, name, archetypeDir, targetFilename, filename)
+		if err != nil {
+			return errors.Wrap(err, "failed to execute archetype template")
+		}
+
+		if err := helpers.SafeWriteToDisk(targetFilename, bytes.NewReader(content), targetFs); err != nil {
+			return errors.Wrap(err, "failed to save results")
+		}
+	}
+
+	jww.FEEDBACK.Println(targetPath, "created")
+
+	return nil
+}
+
+type archetypeMap struct {
+	// These needs to be parsed and executed as Go templates.
+	contentFiles []*hugofs.LanguageFileInfo
+	// These are just copied to destination.
+	otherFiles []*hugofs.LanguageFileInfo
+	// If the templates needs a fully built site. This can potentially be
+	// expensive, so only do when needed.
+	siteUsed bool
+}
+
+func mapArcheTypeDir(
+	ps *helpers.PathSpec,
+	fs afero.Fs,
+	archetypeDir string) (archetypeMap, error) {
+
+	var m archetypeMap
+
+	walkFn := func(filename string, fi os.FileInfo, err error) error {
+
+		if err != nil {
+			return err
+		}
+
+		if fi.IsDir() {
+			return nil
+		}
+
+		fil := fi.(*hugofs.LanguageFileInfo)
+
+		if hugolib.IsContentFile(filename) {
+			m.contentFiles = append(m.contentFiles, fil)
+			if !m.siteUsed {
+				m.siteUsed, err = usesSiteVar(fs, filename)
+				if err != nil {
+					return err
+				}
+			}
+			return nil
+		}
+
+		m.otherFiles = append(m.otherFiles, fil)
+
+		return nil
+	}
+
+	if err := helpers.SymbolicWalk(fs, archetypeDir, walkFn); err != nil {
+		return m, errors.Wrapf(err, "failed to walk archetype dir %q", archetypeDir)
+	}
+
+	return m, nil
+}
+
+func usesSiteVar(fs afero.Fs, filename string) (bool, error) {
+	f, err := fs.Open(filename)
+	if err != nil {
+		return false, errors.Wrap(err, "failed to open archetype file")
+	}
+	defer f.Close()
+	return helpers.ReaderContains(f, []byte(".Site")), nil
+}
+
+// Resolve the target content path.
+func resolveContentPath(sites *hugolib.HugoSites, fs afero.Fs, targetPath string) (string, *hugolib.Site) {
+	targetDir := filepath.Dir(targetPath)
+	first := sites.Sites[0]
+
+	var (
+		s              *hugolib.Site
+		siteContentDir string
+	)
+
+	// Try the filename: my-post.en.md
+	for _, ss := range sites.Sites {
+		if strings.Contains(targetPath, "."+ss.Language.Lang+".") {
+			s = ss
+			break
+		}
+	}
+
+	for _, ss := range sites.Sites {
+		contentDir := ss.PathSpec.ContentDir
+		if !strings.HasSuffix(contentDir, helpers.FilePathSeparator) {
+			contentDir += helpers.FilePathSeparator
+		}
+		if strings.HasPrefix(targetPath, contentDir) {
+			siteContentDir = ss.PathSpec.ContentDir
+			if s == nil {
+				s = ss
+			}
+			break
+		}
+	}
+
+	if s == nil {
+		s = first
+	}
+
+	if targetDir != "" && targetDir != "." {
+		exists, _ := helpers.Exists(targetDir, fs)
+
+		if exists {
+			return targetPath, s
+		}
+	}
+
+	if siteContentDir != "" {
+		pp := filepath.Join(siteContentDir, strings.TrimPrefix(targetPath, siteContentDir))
+		return s.PathSpec.AbsPathify(pp), s
+
+	} else {
+		return s.PathSpec.AbsPathify(filepath.Join(first.PathSpec.ContentDir, targetPath)), s
+	}
+
+}
+
+// FindArchetype takes a given kind/archetype of content and returns the path
+// to the archetype in the archetype filesystem, blank if none found.
+func findArchetype(ps *helpers.PathSpec, kind, ext string) (outpath string, isDir bool) {
+	fs := ps.BaseFs.Archetypes.Fs
+
+	var pathsToCheck []string
+
+	if kind != "" {
+		pathsToCheck = append(pathsToCheck, kind+ext)
+	}
+	pathsToCheck = append(pathsToCheck, "default"+ext, "default")
+
+	for _, p := range pathsToCheck {
+		fi, err := fs.Stat(p)
+		if err == nil {
+			return p, fi.IsDir()
+		}
+	}
+
+	return "", false
 }

create/content_template_handler.go

@@ -16,9 +16,12 @@ package create
 import (
 	"bytes"
 	"fmt"
+	"path/filepath"
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/source"
 
@@ -41,7 +44,13 @@ type ArchetypeFileData struct {
 	// used in the archetype template. Also, if this is a multilingual setup,
 	// this site is the site that best matches the target content file, based
 	// on the presence of language code in the filename.
-	Site *hugolib.Site
+	Site *hugolib.SiteInfo
+
+	// Name will in most cases be the same as TranslationBaseName, e.g. "my-post".
+	// But if that value is "index" (bundles), the Name is instead the owning folder.
+	// This is the value you in most cases would want to use to construct the title in your
+	// archetype template.
+	Name string
 
 	// The target content file. Note that the .Content will be empty, as that
 	// has not been created yet.
@@ -51,7 +60,7 @@ type ArchetypeFileData struct {
 const (
 	// ArchetypeTemplateTemplate is used as initial template when adding an archetype template.
 	ArchetypeTemplateTemplate = `---
-title: "{{ replace .TranslationBaseName "-" " " | title }}"
+title: "{{ replace .Name "-" " " | title }}"
 date: {{ .Date }}
 draft: true
 ---
@@ -73,7 +82,7 @@ var (
 		"%}x}", "%}}")
 )
 
-func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFilename string) ([]byte, error) {
+func executeArcheTypeAsTemplate(s *hugolib.Site, name, kind, targetPath, archetypeFilename string) ([]byte, error) {
 
 	var (
 		archetypeContent  []byte
@@ -81,23 +90,33 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFile
 		err               error
 	)
 
-	sp := source.NewSourceSpec(s.Deps.Cfg, s.Deps.Fs)
-	f := sp.NewFileInfo("", targetPath, nil)
+	f := s.SourceSpec.NewFileInfo("", targetPath, false, nil)
+
+	if name == "" {
+		name = f.TranslationBaseName()
+
+		if name == "index" || name == "_index" {
+			// Page bundles; the directory name will hopefully have a better name.
+			dir := strings.TrimSuffix(f.Dir(), helpers.FilePathSeparator)
+			_, name = filepath.Split(dir)
+		}
+	}
 
 	data := ArchetypeFileData{
 		Type: kind,
 		Date: time.Now().Format(time.RFC3339),
+		Name: name,
 		File: f,
-		Site: s,
+		Site: &s.Info,
 	}
 
 	if archetypeFilename == "" {
 		// TODO(bep) archetype revive the issue about wrong tpl funcs arg order
 		archetypeTemplate = []byte(ArchetypeTemplateTemplate)
 	} else {
-		archetypeTemplate, err = afero.ReadFile(s.Fs.Source, archetypeFilename)
+		archetypeTemplate, err = afero.ReadFile(s.BaseFs.Archetypes.Fs, archetypeFilename)
 		if err != nil {
-			return nil, fmt.Errorf("Failed to read archetype file %q: %s", archetypeFilename, err)
+			return nil, fmt.Errorf("failed to read archetype file %s", err)
 		}
 
 	}
@@ -110,27 +129,18 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFile
 	templateHandler := s.Deps.Tmpl.(tpl.TemplateHandler)
 	templateName := "_text/" + helpers.Filename(archetypeFilename)
 	if err := templateHandler.AddTemplate(templateName, string(archetypeTemplate)); err != nil {
-		return nil, fmt.Errorf("Failed to parse archetype file %q: %s", archetypeFilename, err)
+		return nil, errors.Wrapf(err, "Failed to parse archetype file %q:", archetypeFilename)
 	}
 
-	templ := templateHandler.Lookup(templateName)
+	templ, _ := templateHandler.Lookup(templateName)
 
 	var buff bytes.Buffer
 	if err := templ.Execute(&buff, data); err != nil {
-		return nil, fmt.Errorf("Failed to process archetype file %q: %s", archetypeFilename, err)
+		return nil, errors.Wrapf(err, "Failed to process archetype file %q:", archetypeFilename)
 	}
 
 	archetypeContent = []byte(archetypeShortcodeReplacementsPost.Replace(buff.String()))
 
-	if !bytes.Contains(archetypeContent, []byte("date")) || !bytes.Contains(archetypeContent, []byte("title")) {
-		// TODO(bep) remove some time in the future.
-		s.Log.FEEDBACK.Println(fmt.Sprintf(`WARNING: date and/or title missing from archetype file %q.
-From Hugo 0.24 this must be provided in the archetype file itself, if needed. Example:
-%s
-`, archetypeFilename, ArchetypeTemplateTemplate))
-
-	}
-
 	return archetypeContent, nil
 
 }

create/content_test.go

@@ -35,8 +35,7 @@ import (
 )
 
 func TestNewContent(t *testing.T) {
-	v := viper.New()
-	initViper(v)
+	assert := require.New(t)
 
 	cases := []struct {
 		kind     string
@@ -49,6 +48,14 @@ func TestNewContent(t *testing.T) {
 		{"stump", "stump/sample-2.md", []string{`title: "Sample 2"`}},      // no archetype file
 		{"", "sample-3.md", []string{`title: "Sample 3"`}},                 // no archetype
 		{"product", "product/sample-4.md", []string{`title = "SAMPLE-4"`}}, // empty archetype front matter
+		{"lang", "post/lang-1.md", []string{`Site Lang: en|Name: Lang 1|i18n: Hugo Rocks!`}},
+		{"lang", "post/lang-2.en.md", []string{`Site Lang: en|Name: Lang 2|i18n: Hugo Rocks!`}},
+		{"lang", "post/lang-3.nn.md", []string{`Site Lang: nn|Name: Lang 3|i18n: Hugo Rokkar!`}},
+		{"lang", "content_nn/post/lang-4.md", []string{`Site Lang: nn|Name: Lang 4|i18n: Hugo Rokkar!`}},
+		{"lang", "content_nn/post/lang-5.en.md", []string{`Site Lang: en|Name: Lang 5|i18n: Hugo Rocks!`}},
+		{"lang", "post/my-bundle/index.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}},
+		{"lang", "post/my-bundle/index.en.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}},
+		{"lang", "post/my-bundle/index.nn.md", []string{`Site Lang: nn|Name: My Bundle|i18n: Hugo Rokkar!`}},
 		{"shortcodes", "shortcodes/go.md", []string{
 			`title = "GO"`,
 			"{{< myshortcode >}}",
@@ -56,39 +63,76 @@ func TestNewContent(t *testing.T) {
 			"{{</* comment */>}}\n{{%/* comment */%}}"}}, // shortcodes
 	}
 
-	for _, c := range cases {
-		cfg, fs := newTestCfg()
-		ps, err := helpers.NewPathSpec(fs, cfg)
-		require.NoError(t, err)
+	for i, c := range cases {
+		cfg, fs := newTestCfg(assert)
+		assert.NoError(initFs(fs))
 		h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs})
-		require.NoError(t, err)
-		require.NoError(t, initFs(fs))
+		assert.NoError(err)
 
-		siteFactory := func(filename string, siteUsed bool) (*hugolib.Site, error) {
-			return h.Sites[0], nil
+		assert.NoError(create.NewContent(h, c.kind, c.path))
+
+		fname := filepath.FromSlash(c.path)
+		if !strings.HasPrefix(fname, "content") {
+			fname = filepath.Join("content", fname)
 		}
-
-		require.NoError(t, create.NewContent(ps, siteFactory, c.kind, c.path))
-
-		fname := filepath.Join("content", filepath.FromSlash(c.path))
 		content := readFileFromFs(t, fs.Source, fname)
-		for i, v := range c.expected {
+		for _, v := range c.expected {
 			found := strings.Contains(content, v)
 			if !found {
-				t.Errorf("[%d] %q missing from output:\n%q", i, v, content)
+				t.Fatalf("[%d] %q missing from output:\n%q", i, v, content)
 			}
 		}
 	}
 }
 
-func initViper(v *viper.Viper) {
-	v.Set("metaDataFormat", "toml")
-	v.Set("archetypeDir", "archetypes")
-	v.Set("contentDir", "content")
-	v.Set("themesDir", "themes")
-	v.Set("layoutDir", "layouts")
-	v.Set("i18nDir", "i18n")
-	v.Set("theme", "sample")
+func TestNewContentFromDir(t *testing.T) {
+	assert := require.New(t)
+	cfg, fs := newTestCfg(assert)
+	assert.NoError(initFs(fs))
+
+	archetypeDir := filepath.Join("archetypes", "my-bundle")
+	assert.NoError(fs.Source.Mkdir(archetypeDir, 0755))
+
+	archetypeThemeDir := filepath.Join("themes", "mytheme", "archetypes", "my-theme-bundle")
+	assert.NoError(fs.Source.Mkdir(archetypeThemeDir, 0755))
+
+	contentFile := `
+File: %s
+Site Lang: {{ .Site.Language.Lang  }} 	
+Name: {{ replace .Name "-" " " | title }}
+i18n: {{ T "hugo" }}
+`
+
+	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755))
+	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "index.nn.md"), []byte(fmt.Sprintf(contentFile, "index.nn.md")), 0755))
+
+	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "pages", "bio.md"), []byte(fmt.Sprintf(contentFile, "bio.md")), 0755))
+	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755))
+	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "resources", "hugo2.xml"), []byte(`hugo2: {{ printf "no template handling in here" }}`), 0755))
+
+	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeThemeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755))
+	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeThemeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755))
+
+	h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs})
+	assert.NoError(err)
+	assert.Equal(2, len(h.Sites))
+
+	assert.NoError(create.NewContent(h, "my-bundle", "post/my-post"))
+
+	assertContains(assert, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/resources/hugo1.json")), `hugo1: {{ printf "no template handling in here" }}`)
+	assertContains(assert, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/resources/hugo2.xml")), `hugo2: {{ printf "no template handling in here" }}`)
+
+	// Content files should get the correct site context.
+	// TODO(bep) archetype check i18n
+	assertContains(assert, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.md")), `File: index.md`, `Site Lang: en`, `Name: My Post`, `i18n: Hugo Rocks!`)
+	assertContains(assert, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.nn.md")), `File: index.nn.md`, `Site Lang: nn`, `Name: My Post`, `i18n: Hugo Rokkar!`)
+
+	assertContains(assert, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/pages/bio.md")), `File: bio.md`, `Site Lang: en`, `Name: My Post`)
+
+	assert.NoError(create.NewContent(h, "my-theme-bundle", "post/my-theme-post"))
+	assertContains(assert, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-theme-post/index.md")), `File: index.md`, `Site Lang: en`, `Name: My Theme Post`, `i18n: Hugo Rocks!`)
+	assertContains(assert, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-theme-post/resources/hugo1.json")), `hugo1: {{ printf "no template handling in here" }}`)
+
 }
 
 func initFs(fs *hugofs.Fs) error {
@@ -131,6 +175,10 @@ title = "{{ .BaseFileName  | upper }}"
 			path:    filepath.Join("archetypes", "emptydate.md"),
 			content: "+++\ndate =\"\"\ntitle = \"Empty Date Arch title\"\ntest = \"test1\"\n+++\n",
 		},
+		{
+			path:    filepath.Join("archetypes", "lang.md"),
+			content: `Site Lang: {{ .Site.Language.Lang  }}|Name: {{ replace .Name "-" " " | title }}|i18n: {{ T "hugo" }}`,
+		},
 		// #3623x
 		{
 			path: filepath.Join("archetypes", "shortcodes.md"),
@@ -165,6 +213,12 @@ Some text.
 	return nil
 }
 
+func assertContains(assert *require.Assertions, v interface{}, matches ...string) {
+	for _, m := range matches {
+		assert.Contains(v, m)
+	}
+}
+
 // TODO(bep) extract common testing package with this and some others
 func readFileFromFs(t *testing.T, fs afero.Fs, filename string) string {
 	filename = filepath.FromSlash(filename)
@@ -184,15 +238,35 @@ func readFileFromFs(t *testing.T, fs afero.Fs, filename string) string {
 	return string(b)
 }
 
-func newTestCfg() (*viper.Viper, *hugofs.Fs) {
+func newTestCfg(assert *require.Assertions) (*viper.Viper, *hugofs.Fs) {
 
-	v := viper.New()
-	fs := hugofs.NewMem(v)
+	cfg := `
 
-	v.SetFs(fs.Source)
+theme = "mytheme"
+	
+[languages]
+[languages.en]
+weight = 1
+languageName = "English"
+[languages.nn]
+weight = 2
+languageName = "Nynorsk"
+contentDir = "content_nn"
 
-	initViper(v)
+`
 
-	return v, fs
+	mm := afero.NewMemMapFs()
+
+	assert.NoError(afero.WriteFile(mm, filepath.Join("i18n", "en.toml"), []byte(`[hugo]
+other = "Hugo Rocks!"`), 0755))
+	assert.NoError(afero.WriteFile(mm, filepath.Join("i18n", "nn.toml"), []byte(`[hugo]
+other = "Hugo Rokkar!"`), 0755))
+
+	assert.NoError(afero.WriteFile(mm, "config.toml", []byte(cfg), 0755))
+
+	v, _, err := hugolib.LoadConfig(hugolib.ConfigSourceDescriptor{Fs: mm, Filename: "config.toml"})
+	assert.NoError(err)
+
+	return v, hugofs.NewFrom(mm, v)
 
 }

deps/deps.go

@@ -1,15 +1,21 @@
 package deps
 
 import (
-	"io/ioutil"
-	"log"
-	"os"
+	"sync"
+	"time"
 
+	"github.com/pkg/errors"
+
+	"github.com/gohugoio/hugo/cache/filecache"
+	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
+	"github.com/gohugoio/hugo/langs"
+	"github.com/gohugoio/hugo/media"
 	"github.com/gohugoio/hugo/metrics"
 	"github.com/gohugoio/hugo/output"
+	"github.com/gohugoio/hugo/resource"
 	"github.com/gohugoio/hugo/source"
 	"github.com/gohugoio/hugo/tpl"
 	jww "github.com/spf13/jwalterweatherman"
@@ -19,12 +25,19 @@ import (
 // There will be normally only one instance of deps in play
 // at a given time, i.e. one per Site built.
 type Deps struct {
+
 	// The logger to use.
-	Log *jww.Notepad `json:"-"`
+	Log *loggers.Logger `json:"-"`
+
+	// Used to log errors that may repeat itself many times.
+	DistinctErrorLog *helpers.DistinctLogger
 
 	// The templates to use. This will usually implement the full tpl.TemplateHandler.
 	Tmpl tpl.TemplateFinder `json:"-"`
 
+	// We use this to parse and execute ad-hoc text templates.
+	TextTmpl tpl.TemplateParseFinder `json:"-"`
+
 	// The file systems to use.
 	Fs *hugofs.Fs `json:"-"`
 
@@ -37,13 +50,19 @@ type Deps struct {
 	// The SourceSpec to use
 	SourceSpec *source.SourceSpec `json:"-"`
 
+	// The Resource Spec to use
+	ResourceSpec *resource.Spec
+
 	// The configuration to use
 	Cfg config.Provider `json:"-"`
 
+	// The file cache to use.
+	FileCaches filecache.Caches
+
 	// The translation func to use
 	Translate func(translationID string, args ...interface{}) string `json:"-"`
 
-	Language *helpers.Language
+	Language *langs.Language
 
 	// All the output formats available for the current site.
 	OutputFormatsConfig output.Formats
@@ -54,6 +73,63 @@ type Deps struct {
 	translationProvider ResourceProvider
 
 	Metrics metrics.Provider
+
+	// Timeout is configurable in site config.
+	Timeout time.Duration
+
+	// BuildStartListeners will be notified before a build starts.
+	BuildStartListeners *Listeners
+
+	*globalErrHandler
+}
+
+type globalErrHandler struct {
+	// Channel for some "hard to get to" build errors
+	buildErrors chan error
+}
+
+// SendErr sends the error on a channel to be handled later.
+// This can be used in situations where returning and aborting the current
+// operation isn't practical.
+func (e *globalErrHandler) SendError(err error) {
+	if e.buildErrors != nil {
+		select {
+		case e.buildErrors <- err:
+		default:
+		}
+		return
+	}
+
+	jww.ERROR.Println(err)
+}
+
+func (e *globalErrHandler) StartErrorCollector() chan error {
+	e.buildErrors = make(chan error, 10)
+	return e.buildErrors
+}
+
+// Listeners represents an event listener.
+type Listeners struct {
+	sync.Mutex
+
+	// A list of funcs to be notified about an event.
+	listeners []func()
+}
+
+// Add adds a function to a Listeners instance.
+func (b *Listeners) Add(f func()) {
+	b.Lock()
+	defer b.Unlock()
+	b.listeners = append(b.listeners, f)
+}
+
+// Notify executes all listener functions.
+func (b *Listeners) Notify() {
+	b.Lock()
+	defer b.Unlock()
+	for _, notify := range b.listeners {
+		notify()
+	}
 }
 
 // ResourceProvider is used to create and refresh, and clone resources needed.
@@ -78,10 +154,6 @@ func (d *Deps) LoadResources() error {
 		return err
 	}
 
-	if th, ok := d.Tmpl.(tpl.TemplateHandler); ok {
-		th.PrintErrors()
-	}
-
 	return nil
 }
 
@@ -107,7 +179,7 @@ func New(cfg DepsCfg) (*Deps, error) {
 	}
 
 	if logger == nil {
-		logger = jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime)
+		logger = loggers.NewErrorLogger()
 	}
 
 	if fs == nil {
@@ -121,24 +193,47 @@ func New(cfg DepsCfg) (*Deps, error) {
 		return nil, err
 	}
 
+	fileCaches, err := filecache.NewCaches(ps)
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to create file caches from configuration")
+	}
+
+	resourceSpec, err := resource.NewSpec(ps, fileCaches, logger, cfg.OutputFormats, cfg.MediaTypes)
+	if err != nil {
+		return nil, err
+	}
+
 	contentSpec, err := helpers.NewContentSpec(cfg.Language)
 	if err != nil {
 		return nil, err
 	}
 
-	sp := source.NewSourceSpec(cfg.Language, fs)
+	sp := source.NewSourceSpec(ps, fs.Source)
+
+	timeoutms := cfg.Language.GetInt("timeout")
+	if timeoutms <= 0 {
+		timeoutms = 3000
+	}
+
+	distinctErrorLogger := helpers.NewDistinctLogger(logger.ERROR)
 
 	d := &Deps{
 		Fs:                  fs,
 		Log:                 logger,
+		DistinctErrorLog:    distinctErrorLogger,
 		templateProvider:    cfg.TemplateProvider,
 		translationProvider: cfg.TranslationProvider,
 		WithTemplate:        cfg.WithTemplate,
 		PathSpec:            ps,
 		ContentSpec:         contentSpec,
 		SourceSpec:          sp,
+		ResourceSpec:        resourceSpec,
 		Cfg:                 cfg.Language,
 		Language:            cfg.Language,
+		FileCaches:          fileCaches,
+		BuildStartListeners: &Listeners{},
+		Timeout:             time.Duration(timeoutms) * time.Millisecond,
+		globalErrHandler:    &globalErrHandler{},
 	}
 
 	if cfg.Cfg.GetBool("templateMetrics") {
@@ -150,10 +245,11 @@ func New(cfg DepsCfg) (*Deps, error) {
 
 // ForLanguage creates a copy of the Deps with the language dependent
 // parts switched out.
-func (d Deps) ForLanguage(l *helpers.Language) (*Deps, error) {
+func (d Deps) ForLanguage(cfg DepsCfg) (*Deps, error) {
+	l := cfg.Language
 	var err error
 
-	d.PathSpec, err = helpers.NewPathSpec(d.Fs, l)
+	d.PathSpec, err = helpers.NewPathSpecWithBaseBaseFsProvided(d.Fs, l, d.BaseFs)
 	if err != nil {
 		return nil, err
 	}
@@ -163,6 +259,15 @@ func (d Deps) ForLanguage(l *helpers.Language) (*Deps, error) {
 		return nil, err
 	}
 
+	// The resource cache is global so reuse.
+	// TODO(bep) clean up these inits.
+	resourceCache := d.ResourceSpec.ResourceCache
+	d.ResourceSpec, err = resource.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.Log, cfg.OutputFormats, cfg.MediaTypes)
+	if err != nil {
+		return nil, err
+	}
+	d.ResourceSpec.ResourceCache = resourceCache
+
 	d.Cfg = l
 	d.Language = l
 
@@ -174,6 +279,8 @@ func (d Deps) ForLanguage(l *helpers.Language) (*Deps, error) {
 		return nil, err
 	}
 
+	d.BuildStartListeners = &Listeners{}
+
 	return &d, nil
 
 }
@@ -184,17 +291,23 @@ func (d Deps) ForLanguage(l *helpers.Language) (*Deps, error) {
 type DepsCfg struct {
 
 	// The Logger to use.
-	Logger *jww.Notepad
+	Logger *loggers.Logger
 
 	// The file systems to use
 	Fs *hugofs.Fs
 
 	// The language to use.
-	Language *helpers.Language
+	Language *langs.Language
 
 	// The configuration to use.
 	Cfg config.Provider
 
+	// The media types configured.
+	MediaTypes media.Types
+
+	// The output formats configured.
+	OutputFormats output.Formats
+
 	// Template handling.
 	TemplateProvider ResourceProvider
 	WithTemplate     func(templ tpl.TemplateHandler) error

docs/.gitignore

@@ -1,3 +1,4 @@
 /.idea
 /public
 nohup.out
+.DS_Store