Add Appveyor builds and webdriver.io tests (tests cover Angular2Spa template only at present)

appveyor.yml

@@ -1,7 +1,20 @@
 init:
   - git config --global core.autocrlf true
 build_script:
-  - build.cmd verify
+  - npm install -g npm@^3.0.0
+  - npm --prefix templates/package-builder install
+  - npm --prefix templates/package-builder run build
+#  - build.cmd verify
 clone_depth: 1
-test: off
+test_script:
+  - dotnet restore ./src
+  - npm install -g selenium-standalone
+  - selenium-standalone install
+  # The nosys flag is needed for selenium to work on Appveyor
+  - ps: Start-Process selenium-standalone 'start','--','-Djna.nosys=true'
+  - npm --prefix test install
+  - npm --prefix test test
+on_finish :
+  # After running tests, upload results to Appveyor
+  - ps: (new-object net.webclient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\test\tmp\junit\*.xml))
 deploy: off

test/.gitignore

@@ -0,0 +1,3 @@
+/node_modules/
+/tmp/
+/yarn.lock

test/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "test",
+  "version": "1.0.0",
+  "description": "Integration tests for the templates in JavaScriptServices. This is not really an NPM package and will not be published.",
+  "main": "index.js",
+  "scripts": {
+    "test": "tsc && wdio"
+  },
+  "author": "Microsoft",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@types/chai": "^3.4.34",
+    "@types/mkdirp": "^0.3.29",
+    "@types/mocha": "^2.2.33",
+    "@types/node": "^6.0.52",
+    "@types/rimraf": "^0.0.28",
+    "@types/webdriverio": "^4.0.32",
+    "chai": "^3.5.0",
+    "cross-spawn": "^5.0.1",
+    "mkdirp": "^0.5.1",
+    "rimraf": "^2.5.4",
+    "selenium-standalone": "^5.9.0",
+    "tree-kill": "^1.1.0",
+    "typescript": "^2.1.4",
+    "webdriverio": "^4.5.0",
+    "yo": "^1.8.5"
+  },
+  "devDependencies": {
+    "wdio-junit-reporter": "^0.2.0",
+    "wdio-mocha-framework": "^0.5.7",
+    "wdio-selenium-standalone-service": "0.0.7"
+  }
+}

test/templates/angular.spec.ts

@@ -0,0 +1,96 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { expect } from 'chai';
+import { generateProjectSync } from './util/yeoman';
+import { AspNetProcess, AspNetCoreEnviroment, defaultUrl, publishProjectSync } from './util/aspnet';
+import { getValue, getCssPropertyValue } from './util/webdriverio';
+
+// First, generate a new project using the locally-built generator-aspnetcore-spa
+// Do this outside the Mocha fixture, otherwise Mocha will time out
+const appDir = path.resolve(__dirname, '../generated/angular');
+const publishedAppDir = path.resolve(appDir, './bin/Release/published');
+if (!process.env.SKIP_PROJECT_GENERATION) {
+    generateProjectSync(appDir, { framework: 'angular-2', name: 'Test App', tests: false });
+    publishProjectSync(appDir, publishedAppDir);
+}
+
+function testBasicNavigation() {
+    describe('Basic navigation', () => {
+        beforeEach(() => browser.url(defaultUrl));
+
+        it('should initially display the home page', () => {
+            expect(browser.getText('h1')).to.eq('Hello, world!');
+            expect(browser.getText('li a[href="https://angular.io/"]')).to.eq('Angular 2');
+        });
+
+        it('should be able to show the counter page', () => {
+            browser.click('a[href="/counter"]');
+            expect(browser.getText('h1')).to.eq('Counter');
+
+            // Test clicking the 'increment' button
+            expect(browser.getText('counter strong')).to.eq('0');
+            browser.click('counter button');
+            expect(browser.getText('counter strong')).to.eq('1');
+        });
+
+        it('should be able to show the fetchdata page', () => {
+            browser.click('a[href="/fetch-data"]');
+            expect(browser.getText('h1')).to.eq('Weather forecast');
+
+            browser.waitForExist('fetchdata table');
+            expect(getValue(browser.elements('fetchdata table tbody tr')).length).to.eq(5);
+        });
+    });
+}
+
+function testHotModuleReplacement() {
+    describe('Hot module replacement', () => {
+        beforeEach(() => browser.url(defaultUrl));
+
+        it('should update when HTML is changed', () => {
+            expect(browser.getText('h1')).to.eq('Hello, world!');
+
+            const filePath = path.resolve(appDir, './ClientApp/app/components/home/home.component.html');
+            const origFileContents = fs.readFileSync(filePath, 'utf8');
+
+            try {
+                const newFileContents = origFileContents.replace('<h1>Hello, world!</h1>', '<h1>HMR is working</h1>');
+                fs.writeFileSync(filePath, newFileContents, { encoding: 'utf8' });
+
+                browser.waitUntil(() => browser.getText('h1').toString() === 'HMR is working');
+            } finally {
+                // Restore old contents so that other tests don't have to account for this
+                fs.writeFileSync(filePath, origFileContents, { encoding: 'utf8' });
+            }
+        });
+
+        it('should update when CSS is changed', () => {
+            expect(getCssPropertyValue(browser, 'li.link-active a', 'color')).to.eq('rgba(255,255,255,1)');
+
+            const filePath = path.resolve(appDir, './ClientApp/app/components/navmenu/navmenu.component.css');
+            const origFileContents = fs.readFileSync(filePath, 'utf8');
+
+            try {
+                const newFileContents = origFileContents.replace('color: white;', 'color: purple;');
+                fs.writeFileSync(filePath, newFileContents, { encoding: 'utf8' });
+
+                browser.waitUntil(() => getCssPropertyValue(browser, 'li.link-active a', 'color') === 'rgba(128,0,128,1)');
+            } finally {
+                // Restore old contents so that other tests don't have to account for this
+                fs.writeFileSync(filePath, origFileContents, { encoding: 'utf8' });
+            }
+        });
+    });
+}
+
+// Now launch dotnet and use selenium to perform tests
+describe('Angular template: dev mode', () => {
+    AspNetProcess.RunInMochaContext(appDir, AspNetCoreEnviroment.development);
+    testBasicNavigation();
+    testHotModuleReplacement();
+});
+
+describe('Angular template: production mode', () => {
+    AspNetProcess.RunInMochaContext(publishedAppDir, AspNetCoreEnviroment.production, 'angular.dll');
+    testBasicNavigation();
+});

test/templates/util/aspnet.ts

@@ -0,0 +1,102 @@
+import * as childProcess from 'child_process';
+import * as path from 'path';
+import * as readline from 'readline';
+const treeKill = require('tree-kill');
+const crossSpawn: typeof childProcess.spawn = require('cross-spawn');
+
+export const defaultUrl = 'http://localhost:5000';
+
+export enum AspNetCoreEnviroment {
+    development,
+    production
+}
+
+export class AspNetProcess {
+    public static RunInMochaContext(cwd: string, mode: AspNetCoreEnviroment, dllToRun?: string) {
+        // Set up mocha before/after callbacks so that a 'dotnet run' process exists
+        // for the same duration as the context this is called inside
+        let aspNetProcess: AspNetProcess;
+        before(() => {
+            aspNetProcess = new AspNetProcess(cwd, mode, dllToRun);
+            return aspNetProcess.waitUntilListening();
+        });
+        after(() => aspNetProcess.dispose());
+    }
+
+    private _process: childProcess.ChildProcess;
+    private _processHasExited: boolean;
+    private _stdoutReader: readline.ReadLine;
+
+    constructor(cwd: string, mode: AspNetCoreEnviroment, dllToRun?: string) {
+        try {
+            // Prepare env for child process. Note that it doesn't inherit parent's env vars automatically,
+            // hence cloning process.env.
+            const childProcessEnv = Object.assign({}, process.env);
+            childProcessEnv.ASPNETCORE_ENVIRONMENT = mode === AspNetCoreEnviroment.development ? 'Development' : 'Production';
+
+            const verbOrAssembly = dllToRun || 'run';
+            console.log(`Running 'dotnet ${ verbOrAssembly }' in ${ cwd }`);
+            this._process = crossSpawn('dotnet', [verbOrAssembly], { cwd: cwd, stdio: 'pipe', env: childProcessEnv });
+            this._stdoutReader = readline.createInterface(this._process.stdout, null);
+
+            // Echo stdout to the test process's own stdout
+            this._stdoutReader.on('line', line => {
+                console.log(`[dotnet] ${ line.toString() }`);
+            });
+
+            // Also echo stderr
+            this._process.stderr.on('data', chunk => {
+                console.log(`[dotnet ERROR] ${ chunk.toString() }`);
+            });
+
+            // Ensure the process isn't orphaned even if Node crashes before we're disposed
+            process.on('exit', () => this._killProcessSync());
+
+            // Also track whether it exited on its own already
+            this._process.on('exit', () => {
+                this._processHasExited = true;
+            });
+        } catch(ex) {
+            console.log('ERROR: ' + ex.toString());
+            throw ex;
+        }
+    }
+
+    public waitUntilListening(): Promise<any> {
+        return new Promise((resolve, reject) => {
+            this._stdoutReader.on('line', (line: string) => {
+                if (line.startsWith('Now listening on:')) {
+                    resolve();
+                }
+            });
+        });
+    }
+
+    public dispose(): Promise<any> {
+        return new Promise((resolve, reject) => {
+            this._killProcessSync(err => {
+                if (err) {
+                    reject(err);
+                } else {
+                    resolve();
+                }
+            });
+        });
+    }
+
+    private _killProcessSync(callback?: (err: any) => void) {
+        if (!this._processHasExited) {
+            // It's important to kill the whole tree, because 'dotnet run' launches a separate 'dotnet exec'
+            // child process that would otherwise be left running
+            treeKill(this._process.pid, 'SIGINT', callback);
+        }
+    }
+}
+
+export function publishProjectSync(sourceDir: string, outputDir: string) {
+    childProcess.execSync(`dotnet publish -c Release -o ${ outputDir }`, {
+        cwd: sourceDir,
+        stdio: 'inherit',
+        encoding: 'utf8'
+    });
+}

test/templates/util/webdriverio.ts

@@ -0,0 +1,12 @@
+// Workaround for missing '.value' property on WebdriverIO.Client<RawResult<T>> that should be of type T
+// Can't notify TypeScript that the property exists directly, because the interface merging feature doesn't
+// appear to support pattern matching in such a way that WebdriverIO.Client<T> is extended only when T
+// itself extends RawResult<U> for some U.
+export function getValue<T>(client: WebdriverIO.Client<WebdriverIO.RawResult<T>>): T {
+    return (client as any).value;
+}
+
+// The official type declarations for getCssProperty are completely wrong. This function matches runtime behaviour.
+export function getCssPropertyValue<T>(client: WebdriverIO.Client<T>, selector: string, cssProperty: string): string {
+    return (client.getCssProperty(selector, cssProperty) as any).value;
+}

test/templates/util/yeoman.ts

@@ -0,0 +1,52 @@
+import * as childProcess from 'child_process';
+import * as path from 'path';
+import * as rimraf from 'rimraf';
+import * as mkdirp from 'mkdirp';
+
+const generatorDirRelative = '../templates/package-builder/dist/generator-aspnetcore-spa';
+const yoPackageDirAbsolute = path.resolve('./node_modules/yo');
+
+export interface GeneratorOptions {
+    framework: string;
+    name: string;
+    tests?: boolean;
+}
+
+export function generateProjectSync(targetDir: string, generatorOptions: GeneratorOptions) {
+    const generatorDirAbsolute = path.resolve(generatorDirRelative);
+    console.log(`Running NPM install to prepare Yeoman generator at ${ generatorDirAbsolute }`);
+    childProcess.execSync(`npm install`, { stdio: 'inherit', cwd: generatorDirAbsolute });
+
+    console.log(`Ensuring empty output directory at ${ targetDir }`);
+    rimraf.sync(targetDir);
+    mkdirp.sync(targetDir);
+
+    const yoExecutableAbsolute = findYeomanCliScript();
+    console.log(`Will invoke Yeoman at ${ yoExecutableAbsolute } to generate application in ${ targetDir } with options:`);
+    console.log(JSON.stringify(generatorOptions, null, 2));
+    const command = `node "${ yoExecutableAbsolute }" "${ path.resolve(generatorDirAbsolute, './app/index.js') }"`;
+    const args = makeYeomanCommandLineArgs(generatorOptions);
+    childProcess.execSync(`${ command } ${ args }`, {
+        stdio: 'inherit',
+        cwd: targetDir
+    });
+}
+
+function findYeomanCliScript() {
+    // On Windows, you can't invoke ./node_modules/.bin/yo from the shell for some reason.
+    // So instead, we'll locate the CLI entrypoint that yeoman would expose if it was installed globally.
+    const yeomanPackageJsonPath = path.join(yoPackageDirAbsolute, './package.json');
+    const yeomanPackageJson = require(yeomanPackageJsonPath);
+    const yeomanCliScriptRelative = yeomanPackageJson.bin.yo;
+    if (!yeomanCliScriptRelative) {
+        throw new Error(`Could not find Yeoman CLI script. Looked for a bin/yo entry in ${ yeomanPackageJsonPath }`);
+    }
+
+    return path.join(yoPackageDirAbsolute, yeomanCliScriptRelative);
+}
+
+function makeYeomanCommandLineArgs(generatorOptions: GeneratorOptions) {
+    return Object.getOwnPropertyNames(generatorOptions)
+        .map(key => `--${ key }="${ generatorOptions[key] }"`)
+        .join(' ');
+}

test/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "moduleResolution": "node",
+    "target": "es5",
+    "rootDir": ".",
+    "outDir": "tmp",
+    "sourceMap": false,
+    "lib": ["es6", "dom"]
+  },
+  "exclude": [
+      "node_modules",
+      "**/node_modules",
+      "tmp"
+  ]
+}

test/wdio.conf.js

@@ -0,0 +1,204 @@
+exports.config = {
+    
+    //
+    // ==================
+    // Specify Test Files
+    // ==================
+    // Define which test specs should run. The pattern is relative to the directory
+    // from which `wdio` was called. Notice that, if you are calling `wdio` from an
+    // NPM script (see https://docs.npmjs.com/cli/run-script) then the current working
+    // directory is where your package.json resides, so `wdio` will be called from there.
+    //
+    specs: [
+        './tmp/templates/**/*.spec.js'
+    ],
+    // Patterns to exclude.
+    exclude: [
+        // 'path/to/excluded/files'
+    ],
+    //
+    // ============
+    // Capabilities
+    // ============
+    // Define your capabilities here. WebdriverIO can run multiple capabilities at the same
+    // time. Depending on the number of capabilities, WebdriverIO launches several test
+    // sessions. Within your capabilities you can overwrite the spec and exclude options in
+    // order to group specific specs to a specific capability.
+    //
+    // First, you can define how many instances should be started at the same time. Let's
+    // say you have 3 different capabilities (Chrome, Firefox, and Safari) and you have
+    // set maxInstances to 1; wdio will spawn 3 processes. Therefore, if you have 10 spec
+    // files and you set maxInstances to 10, all spec files will get tested at the same time
+    // and 30 processes will get spawned. The property handles how many capabilities
+    // from the same test should run tests.
+    //
+    maxInstances: 10,
+    //
+    // If you have trouble getting all important capabilities together, check out the
+    // Sauce Labs platform configurator - a great tool to configure your capabilities:
+    // https://docs.saucelabs.com/reference/platforms-configurator
+    //
+    capabilities: [{
+        // maxInstances can get overwritten per capability. So if you have an in-house Selenium
+        // grid with only 5 firefox instances available you can make sure that not more than
+        // 5 instances get started at a time.
+        maxInstances: 5,
+        //
+        browserName: 'chrome'
+    }],
+    //
+    // ===================
+    // Test Configurations
+    // ===================
+    // Define all options that are relevant for the WebdriverIO instance here
+    //
+    // By default WebdriverIO commands are executed in a synchronous way using
+    // the wdio-sync package. If you still want to run your tests in an async way
+    // e.g. using promises you can set the sync option to false.
+    sync: true,
+    //
+    // Level of logging verbosity: silent | verbose | command | data | result | error
+    logLevel: 'silent',
+    //
+    // Enables colors for log output.
+    coloredLogs: true,
+    //
+    // If you only want to run your tests until a specific amount of tests have failed use
+    // bail (default is 0 - don't bail, run all tests).
+    bail: 0,
+    //
+    // Saves a screenshot to a given path if a command fails.
+    screenshotPath: './tmp/errorShots/',
+    //
+    // Set a base URL in order to shorten url command calls. If your url parameter starts
+    // with "/", then the base url gets prepended.
+    baseUrl: 'http://localhost:5000',
+    //
+    // Default timeout for all waitFor* commands.
+    waitforTimeout: 10000,
+    //
+    // Default timeout in milliseconds for request
+    // if Selenium Grid doesn't send response
+    connectionRetryTimeout: 90000,
+    //
+    // Default request retries count
+    connectionRetryCount: 3,
+    //
+    // Initialize the browser instance with a WebdriverIO plugin. The object should have the
+    // plugin name as key and the desired plugin options as properties. Make sure you have
+    // the plugin installed before running any tests. The following plugins are currently
+    // available:
+    // WebdriverCSS: https://github.com/webdriverio/webdrivercss
+    // WebdriverRTC: https://github.com/webdriverio/webdriverrtc
+    // Browserevent: https://github.com/webdriverio/browserevent
+    // plugins: {
+    //     webdrivercss: {
+    //         screenshotRoot: 'my-shots',
+    //         failedComparisonsRoot: 'diffs',
+    //         misMatchTolerance: 0.05,
+    //         screenWidth: [320,480,640,1024]
+    //     },
+    //     webdriverrtc: {},
+    //     browserevent: {}
+    // },
+    //
+    // Test runner services
+    // Services take over a specific job you don't want to take care of. They enhance
+    // your test setup with almost no effort. Unlike plugins, they don't add new
+    // commands. Instead, they hook themselves up into the test process.
+    // services: ['selenium-standalone'],
+    //
+    // Framework you want to run your specs with.
+    // The following are supported: Mocha, Jasmine, and Cucumber
+    // see also: http://webdriver.io/guide/testrunner/frameworks.html
+    //
+    // Make sure you have the wdio adapter package for the specific framework installed
+    // before running any tests.
+    framework: 'mocha',
+    //
+    // Test reporter for stdout.
+    // The only one supported by default is 'dot'
+    // see also: http://webdriver.io/guide/testrunner/reporters.html
+    reporters: ['junit'],
+    reporterOptions: {
+        outputDir: './tmp/junit'
+    },
+
+    //
+    // Options to be passed to Mocha.
+    // See the full list at http://mochajs.org/
+    mochaOpts: {
+        ui: 'bdd',
+        timeout: 30000
+    },
+    //
+    // =====
+    // Hooks
+    // =====
+    // WebdriverIO provides several hooks you can use to interfere with the test process in order to enhance
+    // it and to build services around it. You can either apply a single function or an array of
+    // methods to it. If one of them returns with a promise, WebdriverIO will wait until that promise got
+    // resolved to continue.
+    //
+    // Gets executed once before all workers get launched.
+    // onPrepare: function (config, capabilities) {
+    // },
+    //
+    // Gets executed just before initialising the webdriver session and test framework. It allows you
+    // to manipulate configurations depending on the capability or spec.
+    // beforeSession: function (config, capabilities, specs) {
+    // },
+    //
+    // Gets executed before test execution begins. At this point you can access all global
+    // variables, such as `browser`. It is the perfect place to define custom commands.
+    // before: function (capabilities, specs) {
+    // },
+    //
+    // Hook that gets executed before the suite starts
+    // beforeSuite: function (suite) {
+    // },
+    //
+    // Hook that gets executed _before_ a hook within the suite starts (e.g. runs before calling
+    // beforeEach in Mocha)
+    // beforeHook: function () {
+    // },
+    //
+    // Hook that gets executed _after_ a hook within the suite starts (e.g. runs after calling
+    // afterEach in Mocha)
+    // afterHook: function () {
+    // },
+    //
+    // Function to be executed before a test (in Mocha/Jasmine) or a step (in Cucumber) starts.
+    // beforeTest: function (test) {
+    // },
+    //
+    // Runs before a WebdriverIO command gets executed.
+    // beforeCommand: function (commandName, args) {
+    // },
+    //
+    // Runs after a WebdriverIO command gets executed
+    // afterCommand: function (commandName, args, result, error) {
+    // },
+    //
+    // Function to be executed after a test (in Mocha/Jasmine) or a step (in Cucumber) starts.
+    // afterTest: function (test) {
+    // },
+    //
+    // Hook that gets executed after the suite has ended
+    // afterSuite: function (suite) {
+    // },
+    //
+    // Gets executed after all tests are done. You still have access to all global variables from
+    // the test.
+    // after: function (result, capabilities, specs) {
+    // },
+    //
+    // Gets executed right after terminating the webdriver session.
+    // afterSession: function (config, capabilities, specs) {
+    // },
+    //
+    // Gets executed after all workers got shut down and the process is about to exit. It is not
+    // possible to defer the end of the process using a promise.
+    // onComplete: function(exitCode) {
+    // }
+}