title:    AWDwR
subtitle: RailsConf
date:     System Testing Rails itself

script:
  - pager.js
stylesheet:
  - presentation.css

watermark: |
  <svg viewBox='0 0 1000 1000' xmlns='http://www.w3.org/2000/svg'>
  <path d='M160,560c2-6-2-13-6-17l-6-9c-104-164,58-356,234-305c59,18,119,63,158,116c33,45,49,95,53,140c8,87-16,169-62,219c0,0-22,24,8,42c28,16,112-18,152-163c37-132-50-403-312-455c-190-37-302,124-304,268c0,58,30,125,63,159c14,14,21,11,22,5' fill='#17733B'/>
  <radialGradient cx='651' cy='363' id='r5g1' r='257' gradientUnits='userSpaceOnUse'>
          <stop offset='0' style='stop-color:#E0EECF'/>
          <stop offset='1' style='stop-color:#17753C'/>
  </radialGradient>
  <path d='M581,733c-20,9-31-2-31-2c107-75,132-334-31-471c-159-133-277-83-327-52c0,0,36-73,145-76c61-2,229,40,314,213c99,202-1,327-22,352c-13,17-37,31-48,36' fill='url(#r5g1)'/>
  <radialGradient cx='477' cy='494' id='r5g2' r='247' gradientUnits='userSpaceOnUse'>
          <stop offset='0' style='stop-color:#DAECCE'/>
          <stop offset='1' style='stop-color:#17733B'/>
  </radialGradient>
  <path d='M562,675c44-90,66-232-27-346c-93-114-198-113-198-113s156-14,250,175c73,147-25,284-25,284' fill='url(#r5g2)'/>
  <path d='M593,219c4,5,12,4,18,4c3-1,7-1,11-1c194-9,279,228,147,354c-45,43-115,72-180,80c-55,6-106-6-147-25c-79-36-139-97-159-163c0,0-10-31-40-14c-29,16-41,106,64,213c96,98,375,159,551-42c127-146,44-324-80-397c-51-30-123-37-169-25c-20,5-21,12-16,16' fill='#821C35'/>
  <radialGradient cx='518' cy='742' id='r1g1' r='257' gradientUnits='userSpaceOnUse'>
          <stop offset='0' style='stop-color:#F7A8A9'/>
          <stop offset='0.7178' style='stop-color:#821C35'/>
  </radialGradient>
  <path d='M232,497c3-22,18-25,18-25c12,129,224,280,424,208c195-71,210-198,208-257c0,0,45,67-7,164c-29,53-149,177-342,164c-223-14-282-164-294-194c-7-20-8-48-7-60' fill='url(#r1g1)'/>
  <radialGradient cx='492' cy='526' id='r1g2' r='247' gradientUnits='userSpaceOnUse'>
          <stop offset='0' style='stop-color:#F2A5B7'/>
          <stop offset='1' style='stop-color:#821C35'/>
  </radialGradient>
  <path d='M293,510c55,83,167,173,312,149c146-24,197-115,197-115s-66,143-276,130c-164-11-233-164-233-164' fill='url(#r1g2)'/>
  <clipPath id="r3c">
          <path d="M999,999 L558,370 c-30-50-40-50-70-84h-400v700z"/>
  </clipPath>
  <g clip-path="url(#r3c)">
  <path d='M677,766c-6,2-10,9-11,14c-2,3-4,7-5,11c-90,172-338,127-381-51c-14-60-5-135,21-195c23-51,58-89,95-115c71-50,154-71,220-56c0,0,32,7,33-28c0-33-71-88-217-51c-133,34-324,245-238,498c62,183,258,200,383,129c51-28,94-88,107-134c5-19-1-23-7-22' fill='#3564AF'/>
  <radialGradient cx='262' cy='440' id='r3g1' r='364' gradientUnits='userSpaceOnUse'>
          <stop offset='0' style='stop-color:#EFF1F6'/>
          <stop offset='1' style='stop-color:#3564AF'/>
  </radialGradient>
  <path d='M617,315c18,14,13,28,13,28c-118-54-355,54-392,263c-36,205,66,281,118,309c0,0-81,6-138-88c-32-51-79-217,28-378c125-186,283-162,316-157c21,3,45,16,55,23' fill='url(#r3g1)'/>
  <radialGradient cx='462' cy='525' id='r3g2' r='350' gradientUnits='userSpaceOnUse'>
          <stop offset='0' style='stop-color:#EFF1F6'/>
          <stop offset='1' style='stop-color:#3564AF'/>
  </radialGradient>
  <path d='M576,362c-100,6-234,58-286,195s1,228,1,228s-90-128,26-304c91-136,259-119,259-119' fill='url(#r3g2)'/>
  </g>
  <radialGradient cx='528' cy='251' id='r6g1' r='277' gradientUnits='userSpaceOnUse'>
          <stop offset='0' style='stop-color:#E2EFCF'/>
          <stop offset='1' style='stop-color:#14743B'/>
  </radialGradient>
  <path d='M804,494c-16,4-38-12-43-43c-17-99-90-288-283-364c-176-69-252-6-252-6c-10,4-6-12,2-18c7-7,105-112,334-13c217,94,278,309,272,383c0,10-3,24-4,29c-5,21-15,28-26,32' fill='url(#r6g1)'/>
  <radialGradient cx='748' cy='240' id='r6g2' r='312' gradientUnits='userSpaceOnUse'>
          <stop offset='0' style='stop-color:#E2EFCF'/>
          <stop offset='1' style='stop-color:#14743B'/>
  </radialGradient>
  <path d='M800,488c-1-132-88-337-289-418c-193-78-256-19-256-19s73-91,293-4c208,82,265,267,278,321c12,52,4,94-4,108c-9,16-22,12-22,12' fill='url(#r6g2)'/>
  <radialGradient cx='634' cy='302' id='r6g3' r='185' gradientUnits='userSpaceOnUse'>
          <stop offset='0' style='stop-color:#E2EFCF'/>
          <stop offset='1' style='stop-color:#14743B'/>
  </radialGradient>
  <path d='M767,457c-16-90-50-158-87-212c-45-66-103-114-191-158c0,0,112,25,198,148c73,106,80,222,80,222' fill='url(#r6g3)'/>
  <path d='M298,243c12,11,9,38-15,58c-78,64-204,222-174,427c28,187,121,221,121,221c8,6-8,11-17,8s-150-35-178-283c-27-235,128-396,195-428c9-4,22-9,27-10c21-6,32-2,41,7' fill='#3564AF'/>
  <radialGradient cx='107' cy='418' id='r4g1' r='441' gradientUnits='userSpaceOnUse'>
          <stop offset='0' style='stop-color:#D3DCE8'/>
          <stop offset='1' style='stop-color:#3564AF'/>
  </radialGradient>
  <path d='M295,248c-114,68-248,245-217,459c29,207,111,232,111,232s-115-18-150-251c-33-221,99-364,140-402c38-36,78-50,95-50c18,0,21,12,21,12' fill='url(#r4g1)'/>
  <radialGradient cx='217' cy='485' id='r4g2' r='261' gradientUnits='userSpaceOnUse'>
          <stop offset='0' style='stop-color:#F0F1F9'/>
          <stop offset='1' style='stop-color:#3564AF'/>
  </radialGradient>
  <path d='M285,293c-70,59-112,121-141,182c-34,71-47,145-41,243c0,0-34-109,29-244c55-117,153-181,153-181' fill='url(#r4g2)'/>
  <radialGradient cx='661' cy='712' id='r2g1' r='275' gradientUnits='userSpaceOnUse'>
          <stop offset='0' style='stop-color:#E89EB0'/>
          <stop offset='1' style='stop-color:#821C35'/>
  </radialGradient>
  <path d='M329,810c4-16,28-27,58-16c94,35,294,66,457-63c147-118,131-216,131-216c1-10,13,2,15,11c2,10,44,147-156,296c-190,141-407,87-468,45l-23-18c-15-16-17-27-14-39' fill='url(#r2g1)'/>
  <radialGradient cx='576' cy='888' id='r2g2' r='312' gradientUnits='userSpaceOnUse'>
          <stop offset='0' style='stop-color:#CFA2AD'/>
          <stop offset='1' style='stop-color:#821C35'/>
  </radialGradient>
  <path d='M335,810c116,64,336,92,506-42c164-128,145-212,145-212s43,108-142,255c-175,139-365,96-418,80c-50-15-83-42-91-57c-9-16,0-24,0-24' fill='url(#r2g2)'/>
  <radialGradient cx='579' cy='758' id='r2g3' r='185' gradientUnits='userSpaceOnUse'>
          <stop offset='0' style='stop-color:#DE95A7'/>
          <stop offset='1' style='stop-color:#821C35'/>
  </radialGradient>
  <path d='M379,796c86,31,161,36,227,31c79-6,150-32,232-87c0,0-78,85-226,98c-129,11-233-42-233-42' fill='url(#r2g3)'/>
  </svg>

#agenda: Agenda

* *Intro*
** "Introduction":intro, 
"System Testing":systest
* Live example
** "rjs":rjs, "render":render, "test-unit":test-unit, "dashboard":dashboard
* Context
** "motivation":motivation, "evolution":evolution
* Components
** "gorp":gorp, "testrails":testrails, "dashboard":dashboard, "cgi-spa":cgi-spa
* Conclusion
** "observation":observation, "conclusion":conclusion, "sources":sources

#intro: Introduction

* My name is Sam Ruby

#intro: Introduction

* My name is Sam Ruby
 * IBM, Emerging Technology
 * Co-author, "Agile Web Development with Rails":http://www.pragprog.com/titles/rails3/agile-web-development-with-rails-third-edition

#intro: Introduction

* My name is Sam Ruby
 * IBM, Emerging Technology
 * Co-author, "Agile Web Development with Rails":http://www.pragprog.com/titles/rails3/agile-web-development-with-rails-third-edition
 * "Weblog":http://intertwingly.net/blog/
 * "More info on Wikipedia":http://en.wikipedia.org/wiki/Sam_Ruby

#systest: System Testing

System testing of software or hardware is testing conducted on a complete,
integrated system to evaluate the system's compliance with its specified
requirements. System testing falls within the scope of black box testing, and
as such, should require no knowledge of the inner design of the code or logic. 

-- "wikipedia":http://en.wikipedia.org/wiki/System_testing

#agenda: Agenda

* Intro
** "Introduction":intro, 
"System Testing":systest
* *Live example*
** "rjs":rjs, "render":render, "test-unit":test-unit, "dashboard":dashboard
* Context
** "motivation":motivation, "evolution":evolution
* Components
** "gorp":gorp, "testrails":testrails, "dashboard":dashboard, "cgi-spa":cgi-spa
* Conclusion
** "observation":observation, "conclusion":conclusion, "sources":sources

#rjs: rjs test output
~06/rjs_functional_test-187.html

# rjs test on 1.9.2
~06/rjs_functional_test-192.html

# rjs test on 1.9.2
~06/rjs_functional_test.rb

# rjs test on 1.9.2
~06/bisect.log

#render: rjs render test output
~07/rjs_render_test.html

# rjs render test on 1.9.2
~07/rjs_render_test.rb

# rjs render test on 1.9.2
~07/bisect.out

#testunit: uninitialized constant on 1.9.2
~08/makedepot.log

# uninitialized constant on 1.9.2
* "reported":http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-core/30688
on ruby-core
* isolated to "revision 28200":http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-core/30688

#dashboard: dashboard
~dashboard.html

#agenda: Agenda

* Intro
** "Introduction":intro, 
"System Testing":systest
* Live example
** "rjs":rjs, "render":render, "test-unit":test-unit, "dashboard":dashboard
* *Context*
** "motivation":motivation, "evolution":evolution
* Components
** "gorp":gorp, "testrails":testrails, "dashboard":dashboard, "cgi-spa":cgi-spa
* Conclusion
** "observation":observation, "conclusion":conclusion, "sources":sources

#motivation: Motivation

How did I get here?

# Motivation

In my case, I did this entirely out of self defense.

# Motivation

In my case, I did this entirely out of self defense.

System testing rails turned out to be a completely unintentional side effect.

# Motivation

_Really, I'm not out to destroy Microsoft. That will just be a completely
unintentional side effect._

- "Linus Torvalds":http://en.wikiquote.org/wiki/Linus_Torvalds

# Motivation

I refuse to be a victim.  

# Motivation

I refuse to be a victim.  

The all too common cycle I see:

# Motivation

I refuse to be a victim.  

The all too common cycle I see:
* Don't test the betas

# Motivation

I refuse to be a victim.  

The all too common cycle I see:
* Don't test the betas
* Wait for a release

# Motivation

I refuse to be a victim.  

The all too common cycle I see:
* Don't test the betas
* Wait for a release
* Complain on blogs and twitter

# Motivation

Don't be a victim.

#evolution: Evolution

I started out with a simple script, in one file, with a micro DSL.

#Evolution

I started out with a simple script, in one file, with a micro DSL.

It ran a few commands, a few HTTP requests, captured the output, and formatted
the results as HTML for visual inspection.

#Evolution

I started out with a simple script, in one file, with a micro DSL.

It ran a few commands, a few HTTP requests, captured the output, and formatted
the results as HTML for visual inspection.

Manually verifying this turned out to be a bit of a drag.

#Evolution

Next step was a separate script to verify the output of the first script.

#Evolution

Next step was a separate script to verify the output of the first script.

It did that by using <code>assert_select</code>.

#Evolution

Once I started work on edition 4, I had need for a second script.

#Evolution

Once I started work on edition 4, I had need for a second script.

I factored out the common code into a library.

#Evolution

Once I started work on edition 4, I had need for a second script.

I factored out the common code into a library.

Best frameworks are factored out of a running application.

#Evolution

Once I started work on edition 4, I had need for a second script.

I factored out the common code into a library.

Best frameworks are factored out of a running application.

This one may turn out to be a good counter-example.

#Evolution

Running two scripts against two versions of Rails and five versions of Ruby
inspired additional automation.

#Evolution

Running two scripts against two versions of Rails and five versions of Ruby
inspired additional automation.

One to set up the directories, use rvm to manage Ruby VMs (even building them
when necessary), use bundler to install dependencies, and to run the script.

#Evolution

Running two scripts against two versions of Rails and five versions of Ruby
inspired additional automation.

One to set up the directories, use rvm to manage Ruby VMs (even building them
when necessary), use bundler to install dependencies, and to run the script.

One to present the overall status, and launch new tests.

#Evolution

Running two scripts against two versions of Rails and five versions of Ruby
inspired additional automation.

One to set up the directories, use rvm to manage Ruby VMs (even building them
when necessary), use bundler to install dependencies, and to run the script.

One to present the overall status, and launch new tests.

The second script used another library that builds on CGI.

#agenda: Agenda

* Intro
** "Introduction":intro, 
"System Testing":systest
* Live example
** "rjs":rjs, "render":render, "test-unit":test-unit, "dashboard":dashboard
* Context
** "motivation":motivation, "evolution":evolution
* *Components*
** "gorp":gorp, "testrails":testrails, "dashboard":dashboard, "cgi-spa":cgi-spa
* Conclusion
** "observation":observation, "conclusion":conclusion, "sources":sources

#recap: List of components
* testrails: select vm, install bundle, run test
* makedepot: run a single scenario (edition 3 or edition 4)
* gorp: common library
* dashboard: display test results, and submit new tests
* cgi-spa: makes cgi more tolerable

#gorp: List of components
* testrails: select vm, install bundle, run test
* makedepot: run a single scenario (edition 3 or edition 4)
* *gorp: common library*
* dashboard: display test results, and submit new tests
* cgi-spa: makes cgi more tolerable

# Gorp

Gorp is the common library

* Name has no special meaning

# Gorp.xhtmlparse
* originally to handle html entities
* monkey patch for 1.8.6 (obsolete)
* nokogiri support
* backports (via monkeypatch) nokogiri API to REXml

# Gorp.Commands (executable)
* db: sqlite3 command
* ruby: run a script
* rake: run a rake
* console: run rails console
* generate: rails generate
* cmd: arbitrary OS command
* irb: interactive ruby
* rails: set up a new application

# Gorp.Commands (admin)
* omit: indicate what steps should be omitted
* section: a block of commands
* overview: description for a section
* note: description for a step
* log: line to output to the screen
* issue: track bug reports

# Gorp.test
* suite: a test suite (enables assert_select)
* input: parses the _output_ of a run
* output: declares where to place the output
* section: matches a section in the input
* collect_stdout: extract the stdout lines (common idiom)
* sort_hash: simplifies comparison of hashes

# Gorp.test (other)
 * cmd, get, post, rake, ruby, console: augmented with assert_select
 * html_fault: captures errors (and matches issues)
 * html_summary: hook on test completion (stops server, produces output)
 * at_exit: additional support for standalone tests
* test: individual step

# Gorp.net
* snap: capture body of a response
* get: does a HTTP get
* post: does a HTTP get (manages cookies, fills in forms)

# Gorp.env
* dump_env
* log
* path (work path)

# Gorp.edit
* unindent
* highlight
* clear_highlights
* clear_all_marks
* mark
* edit
* msub
* all=
* read
* dcl

# Other:
* restart_server, stop_server: both do what you would expect
* verify that $PORT (defaults to 3000) is available
* parse command for <code>_n.n_</code> syntax
* --rails-debug (ENV['BACKTRACE'] = '1')
* output: produce test output, using builder.  Runs sections, captures errors.
* which_rails: rails command to use (git, gem)
* Commands know about OS and Rails version changes

# Obsolete:
* alternatives to bundler (rails.env)

# Gorp.Globals
* $x: builder
* $toc: table of contents
* $todos: list of todos
* $issue: current issue number
* $style: CSS to be output

#testrails: List of components
* *testrails*: select vm, install bundle, run test
* makedepot: run a single scenario (edition 3 or edition 4)
* gorp: common library
* dashboard: display test results, and submit new tests
* cgi-spa: makes cgi more tolerable

# Testrails
* update Rails from git
* update tests from git
* build a new rvm, if necessary
* update gems using bundler
* run the script
* paginate the output

#dashboard: List of components
* testrails: select vm, install bundle, run test
* makedepot: run a single scenario (edition 3 or edition 4)
* gorp: common library
* *dashboard*: display test results, and submit new tests
* cgi-spa: makes cgi more tolerable

# Dashboard
* displays colorized status in tabular form
* submits new tests in background
* displays live status of running test (AJAX)
* configured via YAML

#cgi-spa: List of components
* testrails: select vm, install bundle, run test
* makedepot: run a single scenario (edition 3 or edition 4)
* gorp: common library
* dashboard: display test results, and submit new tests
* *cgi-spa*: makes cgi more tolerable

# cgi-spa
* methods: json, json!, html, html!, post, post!
* html: style!, script!
* environment: $HTTP_GET, $HTTP_POST, $XHR_JSON, $XHTML, $param, $HOME, $SERVER
* builder: indented_text, indented_data, traceback
* submit
* installation

#agenda: Agenda

* Intro
** "Introduction":intro, 
"System Testing":systest
* Live example
** "rjs":rjs, "render":render, "test-unit":test-unit, "dashboard":dashboard
* Context
** "motivation":motivation, "evolution":evolution
* Components
** "gorp":gorp, "testrails":testrails, "dashboard":dashboard, "cgi-spa":cgi-spa
* *Conclusion*
** "observation":observation, "conclusion":conclusion, "sources":sources

#observation: Observation

Edge of a major new release is atypical

# Observation

Edge of a major new release is atypical

More common errors to be found:
* rails server won't start
* rake test won't test
* permutations not covered by unit tests

#conclusion: Conclusion

Don't be a victim

# Conclusion

Don't be a victim
* incorporate edge Rails into your CI

# Conclusion

Don't be a victim
* incorporate edge Rails into your CI
* report what you find early

#sources: Sources
* "testrails":http://github.com/rubys/awdwr/blob/master/testrails.rb: select vm, install bundle, run test
* "makedepot":http://github.com/rubys/awdwr/blob/master/makedepot.rb: run a single scenario (edition 3)
* "checkdepot":http://github.com/rubys/awdwr/blob/master/checkdepot.rb: verify
* output of a single scenario (edition 3)
* "gorp":http://github.com/rubys/gorp: common library
* "dashboard":http://github.com/rubys/awdwr/blob/master/dashboard.rb: display test results, and submit new tests
* "cgi-spa":http://github.com/rubys/cgi-spa: makes cgi more tolerable

#Thanks: Thanks!

Any questions?