Latest Threat Research:SANDWORM_MODE: Shai-Hulud-Style npm Worm Hijacks CI Workflows and Poisons AI Toolchains.Details →
Socket
Book a DemoInstallSign in
Socket
Book a DemoInstallSign in

twill

Package Overview
Dependencies
Maintainers
2
Versions
33
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

twill - npm Package Compare versions

Comparing version
3.0.3
 to
3.1
+16
tests/test_show.py
from pytest import raises
from twill import commands
from twill.errors import TwillException
from .utils import execute_script
def test(url):
commands.show()
commands.show('html')
commands.show('links')
with raises(TwillException, match='Cannot show "nonsense".'):
commands.show('nonsense')
execute_script('test_show.twill', initial_url=url)
tests/test_show.twill

Sorry, the diff of this file is not supported yet

+44
tests/test_two_forms.py
from pytest import raises
from twill import commands
from twill.errors import TwillException
def test(url):
commands.go('/two_forms')
commands.find(' NO FORM ')
with raises(TwillException):
commands.submit()
with raises(TwillException):
commands.submit('1')
commands.fv('1', 'item', 'foo')
commands.submit()
commands.find(' FORM=1 ITEM=foo ')
commands.fv('2', 'item', 'bar')
commands.submit()
commands.find(' FORM=2 ITEM=bar ')
with raises(TwillException):
commands.submit()
commands.submit('1', '1')
commands.find(' FORM=1 ITEM= ')
commands.submit('1', '2')
commands.find(' FORM=2 ITEM= ')
with raises(TwillException):
commands.submit('1', '3')
commands.fv('1', 'item', 'foo')
commands.fv('2', 'item', 'bar')
commands.submit()
commands.find(' FORM=2 ITEM=bar ')
commands.fv('2', 'item', 'bar')
commands.fv('1', 'item', 'foo')
commands.submit()
commands.find(' FORM=1 ITEM=foo ')
+216
twill/agents.py
"""
Map of various User-Agent string shortcuts that can be used for testing.
"""
from typing import Dict
# noinspection HttpUrlsUsage
agents: Dict[str, str] = dict(
# Desktop
chrome_40='Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36'
' (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36',
chrome_107='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
' (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',
edge_12='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
' (KHTML, like Gecko) Chrome/42.0.2311.135'
' Safari/537.36 Edge/12.246',
edge_107='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
' (KHTML, like Gecko) Chrome/107.0.0.0'
' Safari/537.36 Edg/107.0.1418.26',
firefox_40='Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0)'
' Gecko/20100101 Firefox/40.1',
firefox_106='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0)'
' Gecko/20100101 Firefox/106.0',
ie_3='Mozilla/2.0 (compatible; MSIE 3.0; Windows 3.1)',
ie_4='Mozilla/4.0 (compatible; MSIE 4.0; Windows NT 5.0)',
ie_5='Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.0)',
ie_6='Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
ie_7='Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
ie_8='Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',
ie_9='Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0)',
ie_10='Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)',
ie_11='Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko',
ie_mobile_9='Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5;'
' Trident/5.0; IEMobile/9.0)',
opera_7='Opera/7.0 (Windows NT 5.1; U) [en]',
opera_8='Opera/8.00 (Windows NT 5.1; U; en)',
opera_9='Opera/9.00 (Windows NT 5.2; U; en)',
opera_10='Opera/9.80 (Windows NT 6.1; U; en) Presto/2.2.15 Version/10.00',
opera_11='Opera/9.80 (Windows NT 6.1; U; en) Presto/2.7.62 Version/11.00',
opera_12='Opera/12.0 (Windows NT 5.1; U; en)'
' Presto/22.9.168 Version/12.00',
opera_mini_7='Opera/9.80 (Android; Opera Mini/7.0.29952/28.2075; en)'
' Presto/2.8.119 Version/11.10',
opera_mini_9='Opera/9.80 (J2ME/MIDP; Opera Mini/9 (Compatible; MSIE:9.0;'
' iPhone; BlackBerry9700; AppleWebKit/24.746; en)'
' Presto/2.5.25 Version/10.54',
konqueror_3='Mozilla/5.0 (compatible; Konqueror/3.0; Linux)',
konqueror_4='Mozilla/5.0 (compatible; Konqueror/4.0; Linux)'
' KHTML/4.0.3 (like Gecko)',
lynx_2_8='Lynx/2.8.7rel.2 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/1.0.0a',
w3m_0_5='w3m/0.5.2 (Linux i686; it; Debian-3.0.6-3)',
netscape_3='Mozilla/3.0 (X11; I; AIX 2)',
netscape_4='Mozilla/4.0 (compatible; Mozilla/5.0 ; Linux i686)',
netscape_4_5='Mozilla/4.5 [en] (X11; I; SunOS 5.6 sun4u)',
netscape_7='Mozilla/5.0 (X11; U; SunOS sun4u; en-US; rv:1.0.1)'
' Gecko/20020921 Netscape/7.0',
netscape_9='Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.8pre)'
' Gecko/20071015 Firefox/2.0.0.7 Navigator/9.0',
palemoon_25='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:25.6)'
' Gecko/20150723 Firefox/31.9 PaleMoon/25.6.0',
safari_1='Mozilla/5.0 (Macintosh; PPC Mac OS X; en)'
' AppleWebKit/85.7 (KHTML, like Gecko) Safari/85.6',
safari_2='Mozilla/5.0 (Macintosh; PPC Mac OS; en)'
' AppleWebKit/412 (KHTML, like Gecko) Safari/412',
safari_3='Mozilla/5.0 (Macintosh; Intel Mac OS X; en)'
' AppleWebKit/522.7 (KHTML, like Gecko) Version/3.0 Safari/522.7',
safari_4='Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_6; en)'
' AppleWebKit/530.9+ (KHTML, like Gecko)'
'Version/4.0 Safari/528.16',
safari_5='Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en)'
' AppleWebKit/534.1+ (KHTML, like Gecko)'
' Version/5.0 Safari/533.16',
safari_6='Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536'
'.26 (KHTML, like Gecko)'
' Version/6.0 Mobile/10A5355d Safari/8536.25',
safari_7='Mozilla/5.0 (iPad; CPU OS 7_1_2 like Mac OS X) AppleWebKit/537'
'.51.2 (KHTML, like Gecko)'
' Version/7.0 Mobile/11D257 Safari/9537.53',
safari_605='Mozilla/5.0 (Macintosh; Intel Mac OS X 13_0)'
' AppleWebKit/605.1.15 (KHTML, like Gecko)'
' Version/16.1 Safari/605.1.15',
vivaldi_5='Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36'
' (KHTML, like Gecko) Chrome/107.0.0.0'
' Safari/537.36 Vivaldi/5.4.2753.51',
# Android phones
galaxy_s7='Mozilla/5.0 (Linux; Android 7.0; SM-G930VC Build/NRD90M; wv)'
' AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0'
' Chrome/58.0.3029.83 Mobile Safari/537.36',
galaxy_s10='Mozilla/5.0 (Linux; Android 9; SM-G973U Build/PPR1.180610.011)'
' AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100'
' Mobile Safari/537.36',
galaxy_s20='Mozilla/5.0 (Linux; Android 10;'
' SM-G980F Build/QP1A.190711.020; wv) AppleWebKit/537.36'
' (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.96'
' Mobile Safari/537.36',
galaxy_s22='Mozilla/5.0 (Linux; Android 12;'
' SM-S906N Build/QP1A.190711.020; wv) AppleWebKit/537.36'
' (KHTML, like Gecko) Version/4.0 Chrome/80.0.3987.119'
' Mobile Safari/537.36',
google_pixel='Mozilla/5.0 (Linux; Android 7.1.1; Google Pixel'
' Build/NMF26F; wv) AppleWebKit/537.36 (KHTML, like Gecko)'
' Version/4.0 Chrome/54.0.2840.85 Mobile Safari/537.36',
google_pixel4='Mozilla/5.0 (Linux; Android 10; Google Pixel 4'
' Build/QD1A.190821.014.C2; wv) AppleWebKit/537.36'
' (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.108'
' Mobile Safari/537.36',
google_pixel_6='Mozilla/5.0 (Linux; Android 12; Pixel 6'
' Build/SD1A.210817.023; wv) AppleWebKit/537.36'
' (KHTML, like Gecko) Version/4.0 Chrome/94.0.4606.71'
' Mobile Safari/537.36',
nexus_6p='Mozilla/5.0 (Linux; Android 6.0.1; Nexus 6P Build/MMB29P)'
' AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.83'
' Mobile Safari/537.36',
sony_xperia_1='Mozilla/5.0 (Linux; Android 9;'
' J8110 Build/55.0.A.0.552; wv) AppleWebKit/537.36'
' (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99'
' Mobile Safari/537.36',
htc_one_x10='Mozilla/5.0 (Linux; Android 6.0; HTC One'
' X10 Build/MRA58K; wv) AppleWebKit/537.36'
' (KHTML, like Gecko) Version/4.0 Chrome/61.0.3163.98'
' Mobile Safari/537.36',
# iPhones
iphone_6='Mozilla/5.0 (Apple-iPhone7C2/1202.466; U; CPU like Mac OS X; en)'
' AppleWebKit/420+ (KHTML, like Gecko) Version/3.0'
' Mobile/1A543 Safari/419.3',
iphone_7='Mozilla/5.0 (iPhone9,3; U; CPU iPhone OS 10_0_1 like Mac OS X)'
' AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0'
' Mobile/14A403 Safari/602.1',
iphone_8='Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X)'
' AppleWebKit/604.1.34 (KHTML, like Gecko) Version/11.0'
' Mobile/15A5341f Safari/604.1',
iphone_x='Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X)'
' AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0'
' Mobile/15A372 Safari/604.1',
iphone_11='Mozilla/5.0 (iPhone12,1; U; CPU iPhone OS 13_0 like Mac OS X)'
' AppleWebKit/602.1.50 (KHTML, like Gecko)'
' Version/10.0 Mobile/15E148 Safari/602.1',
iphone_12='Mozilla/5.0 (iPhone13,2; U; CPU iPhone OS 14_0 like Mac OS X)'
' AppleWebKit/602.1.50 (KHTML, like Gecko)'
' Version/10.0 Mobile/15E148 Safari/602.1',
iphone_13_pro_max='Mozilla/5.0 (iPhone14,3; U; CPU iPhone OS 15_0'
' like Mac OS X) AppleWebKit/602.1.50'
' (KHTML, like Gecko) Version/10.0'
' Mobile/19A346 Safari/602.1',
iphone_se_3='Mozilla/5.0 (iPhone14,6; U; CPU iPhone OS 15_4'
' like Mac OS X) AppleWebKit/602.1.50'
' (KHTML, like Gecko) Version/10.0 Mobile/19E241 Safari/602.1',
# MS Windows phones
ms_lumia_650='Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft;'
' RM-1152) AppleWebKit/537.36 (KHTML, like Gecko)'
' Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254',
ms_lumia_950='Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft;'
' Lumia 950) AppleWebKit/537.36 (KHTML, like Gecko)'
' Chrome/46.0.2486.0 Mobile Safari/537.36 Edge/13.1058',
# Tablets
galaxy_tab_s8='Mozilla/5.0 (Linux; Android 12;'
' SM-X906C Build/QP1A.190711.020; wv) AppleWebKit/537.36'
' (KHTML, like Gecko) Version/4.0 Chrome/80.0.3987.119'
' Mobile Safari/537.36',
lenovo_yoga_tab_11='Mozilla/5.0 (Linux; Android 11; Lenovo YT-J706X)'
' AppleWebKit/537.36 (KHTML, like Gecko)'
' Chrome/96.0.4664.45 Safari/537.36',
sony_xperia_tab_z4='Mozilla/5.0 (Linux; Android 6.0.1;'
' SGP771 Build/32.2.A.0.253; wv) AppleWebKit/537.36'
' (KHTML, like Gecko) Version/4.0'
' Chrome/52.0.2743.98 Safari/537.36',
galaxy_tab_s3='Mozilla/5.0 (Linux; Android 7.0; SM-T827R4 Build/NRD90M)'
' AppleWebKit/537.36 (KHTML, like Gecko)'
' Chrome/60.0.3112.116 Safari/537.36',
amazon_fire_hdx_7='Mozilla/5.0 (Linux; Android 4.4.3; KFTHWI Build/KTU84M)'
' AppleWebKit/537.36 (KHTML, like Gecko) Silk/47.1.79'
' like Chrome/47.0.2526.80 Safari/537.36',
lg_g_pad_7='Mozilla/5.0 (Linux; Android 5.0.2; LG-V410/V41020c'
' Build/LRX22G) AppleWebKit/537.36 (KHTML, like Gecko)'
' Version/4.0 Chrome/34.0.1847.118 Safari/537.36',
# E-Readers
kindle_4='Mozilla/5.0 (X11; U; Linux armv7l like Android; en-us)'
' AppleWebKit/531.2+ (KHTML, like Gecko) Version/5.0'
' Safari/533.2+ Kindle/3.0+',
kindle_3='Mozilla/5.0 (Linux; U; en-US) AppleWebKit/528.5+'
' (KHTML, like Gecko, Safari/528.5+) Version/4.0 Kindle/3.0'
' (screen 600x800; rotate)',
# Set tops
chromecast='Mozilla/5.0 (CrKey armv7l 1.5.16041) AppleWebKit/537.36'
' (KHTML, like Gecko) Chrome/31.0.1650.0 Safari/537.36',
amazon_4k_fire_tv='Mozilla/5.0 (Linux; Android 5.1; AFTS Build/LMY47O)'
' AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0'
' Chrome/41.99900.2250.0242 Safari/537.36',
nexus_player='Dalvik/2.1.0 (Linux; U; Android 6.0.1;'
' Nexus Player Build/MMB29T)',
apple_tv_6='AppleTV11,1/11.1',
apple_tv_5='AppleTV6,2/11.1',
apple_tv_4='AppleTV5,3/9.1.1',
# Game consoles
playstation_5='Mozilla/5.0 (PlayStation; PlayStation 5/2.26)'
' AppleWebKit/605.1.15 (KHTML, like Gecko)'
' Version/13.0 Safari/605.1.15',
playstation_4='Mozilla/5.0 (PlayStation 4 3.11) AppleWebKit/537.73'
' (KHTML, like Gecko)',
xbox_x='Mozilla/5.0 (Windows NT 10.0; Win64; x64; Xbox; Xbox Series X)'
' AppleWebKit/537.36 (KHTML, like Gecko)'
' Chrome/48.0.2564.82 Safari/537.36 Edge/20.02',
xbox_one='Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Xbox; Xbox One)'
' AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0'
' Mobile Safari/537.36 Edge/13.10586',
nintendo_switch='Mozilla/5.0 (Nintendo Switch; WifiWebAuthApplet)'
' AppleWebKit/601.6 (KHTML, like Gecko) NF/4.0.0.5.10'
' NintendoBrowser/5.1.0.13343',
# Bots
google_bot_2='Mozilla/5.0 (compatible; Googlebot/2.1;'
' +http://www.google.com/bot.html)',
bing_bot_2='Mozilla/5.0 (compatible; bingbot/2.0;'
' +http://www.bing.com/bingbot.htm)',
yahoo_bot='Mozilla/5.0 (compatible; Yahoo! Slurp;'
' http://help.yahoo.com/help/us/ysearch/slurp)',
)
+1
-1
.bumpversion.cfg
[bumpversion]
current_version = 3.0.3
current_version = 3.1
commit = False

@@ -4,0 +4,0 @@ tag = False

+21
-0
docs/changelog.rst

@@ -7,2 +7,23 @@ .. _changelog:

3.1 (released 2022-10-30)
-------------------------
* The submit command now takes an additional parameter to specify a form
that can be used in rare cases when there are no form fields (#7).
* Most commands do not return values any more, they are just commands.
If you are using twill from Python, you should check browser properties
like 'forms' or 'url' instead of using the return values of commands
like 'show_forms' or 'back' (see #13, #14).
* Two-word commands now consistently have underscores in their names,
(e.g. 'form_action', 'get_input', 'show_links'). However, for convenience
and backward compatibility, you can still use the names without underscores
(e.g. 'formaction', 'getinput', 'showlinks'), and the old two-letter
abbreviations (e.g. 'fa' for 'form_action') (#13, #14).
* Instead of 'showforms' or 'show_forms' you can now also write 'show forms',
and similarly for 'cookies', 'links', 'history' and 'html'. The command
'show html' does the same as 'show' without any arguments.
* Renamed shortcuts for user agent strings, and added some more existing ones.
* Added type hints (#15).
* Support Python 3.11.
* Many minor fixes and improvements.
3.0.3 (released 2022-10-12)

@@ -9,0 +30,0 @@ ---------------------------

+104
-61
docs/commands.rst

@@ -13,14 +13,12 @@ .. _commands:

**go** *<url>* -- visit the given URL. The Python function returns the
final URL visited, after all redirects.
**go** *<url>* -- visit the given URL.
**back** -- return to the previous URL. The Python function returns that
URL, if any.
**back** -- return to the previous URL.
**reload** -- reload the current URL. The Python function returns that URL,
if any.
**reload** -- reload the current URL.
**follow** *<link name>* -- follow the given link. The Python function
returns the final URL visited, after all redirects.
**follow** *<link name>* -- follow the given link.
Note: When used from Python, you get the new URL after executing
these commands (after possible redirects) with ``browser.url``.

@@ -33,26 +31,31 @@ Assertions

**find** *<regex>* [*<flags>*] -- assert that the page contains this
regular expression. The variable ``__match__`` is set to the first matching
subgroup (or the entire matching string, if no subgroups are specified).
When called from Python, the matching string is returned. It also accepts
some flags: **i**, for case-insensitive matching, **m** for multi-line mode
and **s** for dotall matching. The flag **x** uses XPath expressions instead
of regular expressions (see below).
**find** *<pattern>* [*<flags>*] -- assert that the page contains the
specified regular expression pattern. The variable ``__match__`` is set to
the first matching subgroup (or the entire matching string, if no subgroups
are specified). When called from Python, the matching string is also returned.
Fhe command also accepts some flags: **i**, for case-insensitive matching,
**m** for multi-line mode and **s** for dotall matching. The flag **x** uses
XPath expressions instead of regular expressions (see below).
**find** *<xpath>* **x** -- assert that the page contains the element
matched by the XPath expression. The variable ``__match__`` is set to
the first matching element's string representation.
the first matching element's text content. When called from Python, the
matching text is also returned.
**notfind** *<regex>* [*<flags>*] -- assert that the page *does not* contain
this regular expression. It accepts the same flags as **find**.
**not_find** *<pattern>* [*<flags>*] -- assert that the page *does not*
contain the specified regular expression pattern. This command accepts the
same flags as **find**.
**notfind** *<xpath>* **x** -- assert that the page *does not* contain this
Alternative spelling: "notfind"
**not_find** *<xpath>* **x** -- assert that the page *does not* contain the
element pointed by the XPath expression.
**url** *<regex>* -- assert that the current URL matches the given regex.
The variable ``__match__`` is set to the first matching subgroup
**url** *<pattern>* -- assert that the current URL matches the given regex
pattern. The variable ``__match__`` is set to the first matching subgroup
(or the entire matching string, if no subgroups are specified).
When called from Python, the matching string is returned.
**title** *<regex>* -- assert that the title of this page matches this
**title** *<pattern>* -- assert that the title of this page matches this
regular expression. The variable ``__match__`` is set to the first matching

@@ -76,45 +79,73 @@ subgroup (or the entire matching string, if no subgroups are specified).

**show** -- show the current page's HTML. When called from Python,
this function will also return a string containing the HTML.
**show** -- show the current page's HTML.
**showlinks** -- show all of the links on the current page.
When called from Python, this function returns a list of the link objects.
Alternative spelling: "showhtml" or "show html"
**showforms** -- show all of the forms on the current page.
When called from Python, this function returns a list of the forms.
**show_links** -- show all of the links on the current page.
**showhistory** -- show the browser history.
When called from Python, this function returns the history.
Alternative spellings: "showlinks" or "show links"
**show_forms** -- show all of the forms on the current page.
Alternative spellings: "showforms" or "show forms"
**show_history** -- show the browser history.
Alternative spellings: "showhistory" or "show history"
**show_cookies** -- show the current cookies.
Alternative spellings: "showcookies" or "show cookies"
Note: When used from Python, you get these objects using the properties
of the browser with the same names, e.g. ``browser.html``, ``browser.links``,
``browser.forms``, ``browser.history`` and ``browser.cookies``.
Forms
=====
**submit** *[<n>]* -- click the n'th submit button, if given;
otherwise submit via the last submission button clicked; if nothing
clicked, use the first submit button on the form. See `details on
form handling`_ for more information.
**submit** *[<button_name>]* -- click the submit button with the given name.
Instead of the name of the button you can also specify the number of the form
element that shall be used as the submit button. If you do not specify a
button name or number, the form is submitted via the last submission button
clicked; if nothing had been clicked, use the first submit button on the form.
See `details on form handling`_ for more information. In rare cases you may
need to specify the form explicitly, you can do so by adding the form name
or number as an additional parameter to the button name or number.
**formvalue** *<formnum> <fieldname> <value>* --- set the given field in
the given form to the given value. For read-only form controls,
**form_value** *<form_name> <field_name> <value>* --- set the given field
in the given form to the given value. For read-only form controls,
the click may be recorded for use by **submit**, but the value is not
changed unless the 'config' command has changed the default behavior.
See 'config' and `details on form handling`_ for more information on
the 'formvalue' command.
the 'form_value' command.
For list controls, you can use 'formvalue <formnum> <fieldname> +value'
or 'formvalue <formnum> <fieldname> -value' to select or deselect a
For list controls, you can use 'form_value <form_name> <field_name> +value'
or 'form_value <form_name> <field_name> -value' to select or deselect a
particular value.
**fv** -- abbreviation for 'formvalue'.
Alternative spellings: "formvalue" or "fv"
**formaction** *<formnum> <action>* -- change the form action URL to the
**fv** -- abbreviation for 'form_value'.
**form_action** *<form_name> <action>* -- change the form action URL to the
given URL.
**fa** -- abbreviation for 'formaction'.
Alternative spellings: "formaction" or "fa"
**formclear** -- clear all values in the form.
**fa** -- abbreviation for 'form_action'.
**formfile** *<formspec> <fieldspec> <filename> [ <content_type> ]* --
**form_clear** -- clear all values in the form.
Alternative spelling: "formclear"
**form_file** *<form_name> <field_name> <filename> [ <content_type> ]* --
attach a file to a file upload button by filename.
Alternative spelling: 'formfile'.
**show_forms** -- show all of the forms on the current page.
Alternative spellings: "showforms" or "show forms"
Cookies

@@ -132,2 +163,4 @@ =======

Alternative spellings: "showcookies" or "show cookies"
Debugging

@@ -144,8 +177,12 @@ =========

**setglobal** *<name> <value>* -- set variable <name> to value <value> in
**set_global** *<name> <value>* -- set variable <name> to value <value> in
global dictionary. The value can be retrieved with '$value'.
**setlocal** *<name> <value>* -- set variable <name> to value <value> in
Alternative spelling: "setglobal"
**set_local** *<name> <value>* -- set variable <name> to value <value> in
local dictionary. The value can be retrieved with '$value'.
Alternative spelling: "setlocal"
The local dictionary is file-specific, while the global module is general

@@ -174,6 +211,8 @@ to all the commands. Local variables will override global variables if

**runfile** *<file1> [ <file2> ... ]* -- execute the given file(s).
**run_file** *<file1> [ <file2> ... ]* -- execute the given file(s).
**rf** -- abbreviation for 'runfile'.
Alternative spellings: "runfile" or "rf"
**rf** -- abbreviation for 'run_file'.
**add_cleanup** *<file1> [ <file2> ... ]* -- add the given cleanup file(s).

@@ -194,9 +233,13 @@ These will be run after the current file has executed (successfully or not).

**getinput** *<prompt>* -- get keyboard input and store it in ``__input__``.
**get_input** *<prompt>* -- get keyboard input and store it in ``__input__``.
When called from Python, this function returns the input value.
**getpassword** *<prompt>* -- get *silent* keyboard input and store
Alternative spelling: "getinput"
**get_password** *<prompt>* -- get *silent* keyboard input and store
it in ``__password__``. When called from Python, this function returns
the input value.
Alternative spelling: "getpassword"
**add_auth** *<realm> <uri> <user> <password>* -- add HTTP Basic

@@ -243,9 +286,9 @@ Authentication information for the given realm/URI combination.

Both the `formvalue` (or `fv`) and `submit` commands rely on a certain
Both the `form_value` (or `fv`) and `submit` commands rely on a certain
amount of implicit cleverness to do their work. In odd situations, it
can be annoying to determine exactly what form field `formvalue` is
going to pick based on your field name, or what form & field `submit`
can be annoying to determine exactly what form field `form_value` is
going to pick based on your field name, or what form and field `submit`
is going to "click" on.
Here is the pseudocode for how `formvalue` and `submit` figure out
Here is the pseudocode for how `form_value` and `submit` figure out
what form to use (function `twill.browser.form`)::

@@ -260,13 +303,13 @@

Here is the pseudocode for how `formvalue` and `submit` figure out
Here is the pseudocode for how `form_value` and `submit` figure out
what form field to use (function `twill.browser.form_field`)::
search current form for control name with exact match to fieldname;
search current form for control name with exact match to field_name;
if single (unique) match, select.
if no match, convert fieldname into a number and use as an index, if
if no match, convert field_name into a number and use as an index, if
possible.
if no match, search current form for control name with regex match to
fieldname; if single (unique) match, select.
field_name; if single (unique) match, select.

@@ -278,3 +321,3 @@ if *still* no match, look for exact matches to submit-button values.

if a form was _not_ previously selected by formvalue:
if a form was _not_ previously selected by form_value:
if there's only one form on the page, select it.

@@ -284,6 +327,6 @@ otherwise, fail.

if a field is not explicitly named:
if a submit button was "clicked" with formvalue, use it.
if a submit button was "clicked" with form_value, use it.
otherwise, use the first submit button on the form, if any.
otherwise:
find the field using the same rules as formvalue
find the field using the same rules as form_value

@@ -290,0 +333,0 @@ finally, if a button has been picked, submit using it;

+3
-3
docs/conf.py

@@ -21,7 +21,7 @@ # Configuration file for the Sphinx documentation builder.

project = 'twill'
copyright = '2022, C. Titus Brown, Ben R. Taylor et al'
author = 'C. Titus Brown, Ben R. Taylor et al'
copyright = '2022, C. Titus Brown, Ben R. Taylor, Christoph Zwerschke et al'
author = 'C. Titus Brown, Ben R. Taylor, Christoph Zwerschke et al'
# The full version, including alpha/beta/rc tags
version = release = '3.0.3'
version = release = '3.1'

@@ -28,0 +28,0 @@ 

+6
-1
docs/developer.rst

@@ -26,3 +26,3 @@ .. _developer:

twill 0.8 and above are licensed under the `MIT license`_.
All code taken from twill 0.8 is Copyright (C) 2005, 2006, 2007
All code taken from twill 0.x is Copyright (C) 2005-2007
C. Titus Brown <titus@idyll.org>.

@@ -34,2 +34,7 @@

Newer versions 1.x, 2.x and 3.x are also Copyright (C) 2007-2022
Ben R. Taylor , Adam V. Brandizzi, Christoph Zwerschke et al.
The newer versions are released under the same `MIT license`_.
.. _MIT license: http://www.opensource.org/licenses/mit-license.php

@@ -36,0 +41,0 @@ 

+12
-4
docs/python-api.rst

@@ -38,7 +38,7 @@ .. _python-api:

However, the functions in ``commands.py`` are too simple for some situations.
In particular, they do not have any return values, so in order to e.g. get
the HTML for a particular page, you will need to talk to the actual "Web
browser" object that twill uses.
In particular, most of them do not have any return values, so in order to
e.g. get the HTML for a particular page, you will need to ask the actual
"Web browser" object that twill uses.
To talk to the Web browser directly, import the ``browser`` object::
To make use of to the Web browser directly, import the ``browser`` object::

@@ -48,2 +48,3 @@ from twill import browser

browser.go("https://www.python.org/")
assert 'Python' in browser.html
browser.showforms()

@@ -59,2 +60,4 @@

go("https://www.python.org/")
assert 'Python' in browser.html
find("Documentation")
browser.showforms()

@@ -67,2 +70,7 @@

Most importantly, the browser object also provides several properties that
can be used to introspect the current state of the browser and evaluated
programmatically in Python, such as ``url``, ``code``, ``html``, ``title``,
``links``, ``forms``, ``cookies``, ``response_headers`` or ``history``.
For more information on the functions exposed by the browser object,

@@ -69,0 +77,0 @@ see the code of the **TwillBrowser** class in twill.browser.

+2
-2
LICENSE
The MIT License, https://opensource.org/licenses/MIT
twill is Copyright (c) 2005, 2006, 2007 by C. Titus Brown.
Copyright 2005-2022 by C. Titus Brown, Ben R. Taylor, Christoph Zwerschke

@@ -21,2 +21,2 @@ Permission is hereby granted, free of charge, to any person obtaining a copy

OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
THE SOFTWARE.
+8
-10
PKG-INFO
Metadata-Version: 2.1
Name: twill
Version: 3.0.3
Version: 3.1
Summary: twill web browsing and testing language and associated utilities.
Home-page: https://github.com/twill-tools/twill
Author: C. Titus Brown, Ben R. Taylor et al.
Download-URL: https://pypi.org/project/twill/
Author: C. Titus Brown, Ben R. Taylor, Christoph Zwerschke et al.
Author-email: titus@idyll.org
Maintainer: C. Titus Brown
Maintainer-email: titus@idyll.org
Maintainer: Christoph Zwerschke
Maintainer-email: cito@online.de
License: MIT
Download-URL: https://pypi.org/project/twill/
Project-URL: Source, https://github.com/twill-tools/twill

@@ -16,3 +16,2 @@ Project-URL: Issues, https://github.com/twill-tools/twill/issues

Project-URL: ChangeLog, https://twill-tools.github.io/twill/changelog.html
Platform: UNKNOWN
Classifier: Development Status :: 6 - Mature

@@ -32,2 +31,3 @@ Classifier: Environment :: Console

Classifier: Programming Language :: Python :: 3.10
Classifier: Programming Language :: Python :: 3.11
Classifier: Programming Language :: Other Scripting Engines

@@ -47,3 +47,3 @@ Classifier: Topic :: Internet :: WWW/HTTP

The current version 3.0.3 supports Python 3.6 to 3.10.
The current version 3.1 supports Python 3.6 to 3.11.

@@ -54,3 +54,3 @@ See also the [changelog](https://twill-tools.github.io/twill/changelog.html) for a summary of the things that have been changed and improved since version 2.0, and the [acknowledgements](https://twill-tools.github.io/twill/overview.html#acknowledgements) for a short overview of the earlier history of twill.

Copyright (c) by C. Titus Brown et al.
Copyright (c) 2005-2022 by C. Titus Brown, Ben R. Taylor, Christoph Zwerschke et al.

@@ -60,3 +60,1 @@ Newer versions have been created and are maintained by [Christoph Zwerschke](https://github.com/Cito).

twill is available for use, modification, and distribution under the MIT license.
+2
-2
README.md

@@ -6,3 +6,3 @@ twill: a simple scripting language for web browsing

The current version 3.0.3 supports Python 3.6 to 3.10.
The current version 3.1 supports Python 3.6 to 3.11.

@@ -13,3 +13,3 @@ See also the [changelog](https://twill-tools.github.io/twill/changelog.html) for a summary of the things that have been changed and improved since version 2.0, and the [acknowledgements](https://twill-tools.github.io/twill/overview.html#acknowledgements) for a short overview of the earlier history of twill.

Copyright (c) by C. Titus Brown et al.
Copyright (c) 2005-2022 by C. Titus Brown, Ben R. Taylor, Christoph Zwerschke et al.

@@ -16,0 +16,0 @@ Newer versions have been created and are maintained by [Christoph Zwerschke](https://github.com/Cito).

+9
-0
setup.cfg
[tool:pytest]
testpaths = tests
[tool:mypy]
python_version = 3.9
check_untyped_defs = true
no_implicit_optional = true
strict_optional = false
warn_redundant_casts = true
warn_unused_ignores = true
disallow_untyped_defs = false
[egg_info]

@@ -5,0 +14,0 @@ tag_build =

+15
-9
setup.py

@@ -14,7 +14,12 @@ #!/usr/bin/env python3

init = init_file.read()
description = re.search('"""(.*)', init).group(1)
version = re.search("__version__ = '(.*)'", init).group(1)
url = re.search("__url__ = '(.*)'", init).group(1)
download_url = re.search("__download_url__ = '(.*)'", init).group(1)
def find(pattern):
match = re.search(pattern, init)
return match.group(1) if match else None
description = find('"""(.*)')
version = find("__version__ = '(.*)'")
url = find("__url__ = '(.*)'")
download_url = find("__download_url__ = '(.*)'")
with open("README.md") as readme_file:

@@ -28,4 +33,4 @@ readme = readme_file.read()

require_wsgi_intercept = ['wsgi_intercept>=1.10,<2']
require_tests = [
'pytest>=7,<8'] + require_tidy + require_quixote + require_wsgi_intercept
require_tests = ['pytest>=7,<7.1'] + (
require_tidy + require_quixote + require_wsgi_intercept)

@@ -43,3 +48,3 @@

author='C. Titus Brown, Ben R. Taylor et al.',
author='C. Titus Brown, Ben R. Taylor, Christoph Zwerschke et al.',
author_email='titus@idyll.org',

@@ -54,4 +59,4 @@

maintainer='C. Titus Brown',
maintainer_email='titus@idyll.org',
maintainer='Christoph Zwerschke',
maintainer_email='cito@online.de',

@@ -83,2 +88,3 @@ long_description=readme,

'Programming Language :: Python :: 3.10',
'Programming Language :: Python :: 3.11',
'Programming Language :: Other Scripting Engines',

@@ -85,0 +91,0 @@ 'Topic :: Internet :: WWW/HTTP',

+1
-1
tests/__init__.py

@@ -1,1 +0,1 @@

# twill test suite
"""twill test suite"""
+1
-1
tests/mock_dns.py

@@ -76,3 +76,3 @@ """Simple mock implementation of dnspython to test the twill dns extension"""

def query(self, qname, qtype='A'):
def resolve(self, qname, qtype='A'):
if self.nameservers:

@@ -79,0 +79,0 @@ raise ValueError(f'unknown name servers: {self.nameservers}')

+33
-8
tests/server.py

@@ -9,8 +9,8 @@ #!/usr/bin/env python3

from quixote.publish import Publisher
from quixote.errors import AccessError
from quixote.session import Session, SessionManager
from quixote.directory import Directory, AccessControlled
from quixote.form import widget
from quixote import (
from quixote.publish import Publisher # type: ignore
from quixote.errors import AccessError # type: ignore
from quixote.session import Session, SessionManager # type: ignore
from quixote.directory import Directory, AccessControlled # type: ignore
from quixote.form import widget # type: ignore
from quixote import ( # type: ignore
get_session, get_session_manager, get_path,

@@ -109,3 +109,3 @@ redirect, get_request, get_response)

"test_checkbox", "test_simple_checkbox", "echo",
"test_checkboxes", 'test_global_form',
"test_checkboxes", 'test_global_form', "two_forms",
'broken_form_1', 'broken_form_2', 'broken_form_3',

@@ -417,2 +417,27 @@ 'broken_form_4', 'broken_form_5', 'broken_linktext',

def two_forms(self):
request = get_request()
if request.form:
form = request.form.get('form')
item = request.form.get('item')
s = f'FORM={form} ITEM={item}'
else:
s = "NO FORM"
return f"""\
<h1>Two Forms</h1>
<p>== {s} ==</p>
<form method=POST id=form1>
<input type=text name=item>
<input type=hidden name=form value=1>
<input type=submit value=post>
</form>
<form method=POST id=form2>
<input type=text name=item>
<input type=hidden name=form value=2>
<input type=submit value=post>
</form>
"""
def test_checkbox(self):

@@ -600,3 +625,3 @@ request = get_request()

if __name__ == '__main__':
from quixote.server.simple_server import run
from quixote.server.simple_server import run # type: ignore
port = int(os.environ.get('TWILL_TEST_PORT', PORT))

@@ -603,0 +628,0 @@ print(f'starting twill test server on port {port}.')

+16
-11
tests/test_misc.py

@@ -10,18 +10,20 @@ """Test a boatload of miscellaneous functionality."""

from twill import browser, commands
from twill.browser import TwillBrowser
from twill.errors import TwillAssertionError, TwillException
from twill.errors import TwillException
def test():
assert isinstance(browser, TwillBrowser)
assert browser is not None
for attr in ('go', 'reset', 'submit'):
assert hasattr(browser, attr)
# reset
commands.reset_browser()
assert isinstance(browser, TwillBrowser)
assert browser is not None
for attr in ('go', 'reset', 'submit'):
assert hasattr(browser, attr)
# check the 'None' value of return code
assert browser.code is None
with raises(TwillException): # no page and thus no status code yet
assert browser.code
# no forms, right?
with raises(TwillException):
with raises(TwillException): # no page and thus no form yet
browser.submit()

@@ -39,6 +41,9 @@

with raises(TwillAssertionError):
commands.reset_browser()
commands.showhistory()
commands.reset_browser()
commands.showhistory()
with raises(TwillException): # no page, cannot tidy yet
commands.tidy_ok()
with raises(TwillException): # no page, cannot show yet
commands.show()

@@ -45,0 +50,0 @@ 

+1
-1
tests/test_unit_support.py

@@ -12,3 +12,3 @@ """

from quixote.server.simple_server import run as quixote_run
from quixote.server.simple_server import run as quixote_run # type: ignore

@@ -15,0 +15,0 @@ PORT = 8081 # default port to run the server on

+1
-1
tests/test_wsgi_intercept.py

@@ -5,3 +5,3 @@ """Test the WSGI intercept code."""

from wsgi_intercept import (
from wsgi_intercept import ( # type: ignore
requests_intercept, add_wsgi_intercept, remove_wsgi_intercept)

@@ -8,0 +8,0 @@ 

+1
-0
tests/utils.py

@@ -122,2 +122,3 @@ """Utility functions for testing twill"""

# noinspection HttpUrlsUsage
_url = f'http://{HOST}:{port}/'

@@ -124,0 +125,0 @@ 

+12
-1
tox.ini
[tox]
envlist = py3{6,7,8,9,10}, flake8, docs, manifest
envlist = py3{6,7,8,9,10,11}, flake8, mypy, docs, manifest

@@ -10,2 +10,13 @@ [testenv:flake8]

[testenv:mypy]
basepython = python3.10
deps =
mypy==0.981
dnspython>=2,<3
types-lxml
types-requests
types-setuptools
commands =
mypy twill tests extras setup.py
[testenv:docs]

@@ -12,0 +23,0 @@ basepython = python3.10

+0
-1
twill.egg-info/entry_points.txt
[console_scripts]
twill = twill.shell:main
twill-fork = twill.fork:main
+8
-10
twill.egg-info/PKG-INFO
Metadata-Version: 2.1
Name: twill
Version: 3.0.3
Version: 3.1
Summary: twill web browsing and testing language and associated utilities.
Home-page: https://github.com/twill-tools/twill
Author: C. Titus Brown, Ben R. Taylor et al.
Download-URL: https://pypi.org/project/twill/
Author: C. Titus Brown, Ben R. Taylor, Christoph Zwerschke et al.
Author-email: titus@idyll.org
Maintainer: C. Titus Brown
Maintainer-email: titus@idyll.org
Maintainer: Christoph Zwerschke
Maintainer-email: cito@online.de
License: MIT
Download-URL: https://pypi.org/project/twill/
Project-URL: Source, https://github.com/twill-tools/twill

@@ -16,3 +16,2 @@ Project-URL: Issues, https://github.com/twill-tools/twill/issues

Project-URL: ChangeLog, https://twill-tools.github.io/twill/changelog.html
Platform: UNKNOWN
Classifier: Development Status :: 6 - Mature

@@ -32,2 +31,3 @@ Classifier: Environment :: Console

Classifier: Programming Language :: Python :: 3.10
Classifier: Programming Language :: Python :: 3.11
Classifier: Programming Language :: Other Scripting Engines

@@ -47,3 +47,3 @@ Classifier: Topic :: Internet :: WWW/HTTP

The current version 3.0.3 supports Python 3.6 to 3.10.
The current version 3.1 supports Python 3.6 to 3.11.

@@ -54,3 +54,3 @@ See also the [changelog](https://twill-tools.github.io/twill/changelog.html) for a summary of the things that have been changed and improved since version 2.0, and the [acknowledgements](https://twill-tools.github.io/twill/overview.html#acknowledgements) for a short overview of the earlier history of twill.

Copyright (c) by C. Titus Brown et al.
Copyright (c) 2005-2022 by C. Titus Brown, Ben R. Taylor, Christoph Zwerschke et al.

@@ -60,3 +60,1 @@ Newer versions have been created and are maintained by [Christoph Zwerschke](https://github.com/Cito).

twill is available for use, modification, and distribution under the MIT license.
+1
-1
twill.egg-info/requires.txt

@@ -10,3 +10,3 @@ lxml<5,>=4.9

[tests]
pytest<8,>=7
pytest<7.1,>=7
pytidylib<0.4,>=0.3

@@ -13,0 +13,0 @@ quixote<4,>=3.6

+4
-0
twill.egg-info/SOURCES.txt

@@ -80,3 +80,6 @@ .bumpversion.cfg

tests/test_shell_fail.twill
tests/test_show.py
tests/test_show.twill
tests/test_tidy.py
tests/test_two_forms.py
tests/test_unit_support.py

@@ -100,2 +103,3 @@ tests/test_unit_support.twill

twill/__main__.py
twill/agents.py
twill/browser.py

@@ -102,0 +106,0 @@ twill/commands.py

+13
-13
twill/__init__.py

@@ -0,1 +1,13 @@

# This file is part of the twill source distribution.
#
# twill is an extensible scriptlet language for testing Web apps,
# available at https://github.com/twill-tools/twill.
#
# Copyright (c) 2005-2022
# by C. Titus Brown, Ben R. Taylor, Christoph Zwerschke et al.
#
# This program and all associated source code files are released under the
# terms of the MIT license; please see the included LICENSE file for more
# information, or go to https://opensource.org/licenses/mit-license.php.
"""twill web browsing and testing language and associated utilities.

@@ -7,14 +19,2 @@

# This file is part of the twill source distribution.
#
# twill is a extensible scriptlet language for testing Web apps,
# available at http://twill.idyll.org/.
#
# Contact author: C. Titus Brown, titus@idyll.org.
#
# This program and all associated source code files are Copyright (C)
# 2005-2007 by C. Titus Brown. It is released under the MIT license;
# please see the included LICENSE.txt file for more information, or
# go to http://www.opensource.org/licenses/mit-license.php.
import logging

@@ -24,3 +24,3 @@ import sys

__version__ = '3.0.3'
__version__ = '3.1'

@@ -27,0 +27,0 @@ __url__ = 'https://github.com/twill-tools/twill'

+213
-139
twill/browser.py

@@ -5,19 +5,39 @@ """This module implements the TwillBrowser."""

import re
from typing import (
cast, Callable, Dict, IO, List, Optional, Sequence, Tuple, Union)
from urllib.parse import urljoin
import requests
import requests.auth
from lxml import html
from requests import Session
from requests.auth import HTTPBasicAuth
from requests.cookies import RequestsCookieJar
from requests.exceptions import InvalidSchema, ConnectionError
from requests.packages.urllib3.exceptions import InsecureRequestWarning
from requests.structures import CaseInsensitiveDict
from . import log, __version__
from .utils import (
print_form, trunc, unique_match, ResultWrapper, _equiv_refresh_interval)
get_equiv_refresh_interval, html_to_tree, print_form, trunc, unique_match,
CheckboxGroup, FieldElement, FormElement, HtmlElement,
InputElement, Link, UrlWithRealm, RadioGroup, Response, ResultWrapper)
from .errors import TwillException
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
__all__ = ['browser']
def _disable_insecure_request_warnings() -> None:
"""Disable insecure request warnings."""
try:
from requests.packages import urllib3 # type: ignore
except ImportError:
import urllib3 # type: ignore
# noinspection PyUnresolvedReferences
insecure_request_warning = urllib3.exceptions.InsecureRequestWarning
urllib3.disable_warnings(insecure_request_warning)
def _set_http_connection_debuglevel(level: int) -> None:
"""Set the debug level for the connection pool."""
from http.client import HTTPConnection
HTTPConnection.debuglevel = level
class TwillBrowser:

@@ -29,5 +49,5 @@ """A simple, stateful browser"""

def __init__(self):
self.result = None
self.last_submit_button = None
self.first_error = None
self.result: Optional[ResultWrapper] = None
self.last_submit_button: Optional[InputElement] = None
self.first_error: Optional[str] = None

@@ -37,2 +57,5 @@ # whether meta refresh will be displayed

# debug level to be used for the connection pool
self._debug_level = 0
# whether the SSL cert will be verified, or can be a ca bundle path

@@ -42,16 +65,16 @@ self.verify = False

# Session stores cookies
self._session = requests.Session()
self._session = Session()
# An lxml FormElement, none until a form is selected
# A lxml FormElement, None until a form is selected
# replaces self._browser.form from mechanize
self._form = None
self._form_files = {}
self._form: Optional[FormElement] = None
self._form_files: Dict[str, IO] = {}
# A dict of HTTPBasicAuth from requests, keyed off URL
self._auth = {}
self._auth: Dict[UrlWithRealm, HTTPBasicAuth] = {}
# callables to be called after each page load.
self._post_load_hooks = []
self._post_load_hooks: List[Callable] = []
self._history = []
self._history: List[ResultWrapper] = []

@@ -61,2 +84,16 @@ # set default headers

def _assert_result_for(self, what: str) -> ResultWrapper:
if not self.result:
raise TwillException(f"Cannot get {what} since there is no page.")
return self.result
@property
def debug_level(self) -> int:
return self._debug_level
@debug_level.setter
def debug_level(self, level: int) -> None:
_set_http_connection_debuglevel(level)
self._debug_level = level
def reset(self):

@@ -67,14 +104,13 @@ """Reset the browser"""

@property
def creds(self):
def creds(self) -> Dict[UrlWithRealm, HTTPBasicAuth]:
"""Get the credentials for basic authentication."""
return self._auth
@creds.setter
def creds(self, creds):
def add_creds(self, url: UrlWithRealm, user: str, password: str) -> None:
"""Set the credentials for basic authentication."""
self._auth[creds[0]] = requests.auth.HTTPBasicAuth(*creds[1])
self._auth[url] = HTTPBasicAuth(user, password)
def go(self, url):
def go(self, url: str) -> None:
"""Visit given URL."""
try_urls = []
try_urls: List[str] = []
if '://' in url:

@@ -91,2 +127,3 @@ try_urls.append(url)

if not url.startswith(('.', '/', '?')):
# noinspection HttpUrlsUsage
try_urls.append(f'http://{url}')

@@ -106,3 +143,3 @@ try_urls.append(f'https://{url}')

def reload(self):
def reload(self) -> None:
"""Tell the browser to reload the current page."""

@@ -112,3 +149,3 @@ self._journey('reload')

def back(self):
def back(self) -> None:
"""Return to previous page, if possible."""

@@ -122,26 +159,24 @@ try:

@property
def code(self):
def code(self) -> int:
"""Get the HTTP status code received for the current page."""
return self.result.http_code if self.result else None
return self._assert_result_for('status code').http_code
@property
def encoding(self):
def encoding(self) -> Optional[str]:
"""Get the encoding used by the server for the current page."""
return self.result.encoding if self.result else None
return None if self.result is None else self.result.encoding
@property
def html(self):
def html(self) -> str:
"""Get the HTML for the current page."""
return self.result.text if self.result else None
return self._assert_result_for('HTML').text
@property
def dump(self):
def dump(self) -> bytes:
"""Get the binary content of the current page."""
return self.result.content if self.result else None
return self._assert_result_for('content dump').content
@property
def title(self):
if self.result is None:
raise TwillException("Error: Getting title with no page")
return self.result.title
def title(self) -> Optional[str]:
return self._assert_result_for('title').title

@@ -153,10 +188,10 @@ @property

def find_link(self, pattern):
"""Find the first link matching the given pattern.
def find_link(self, pattern: str) -> Optional[Link]:
"""Find the first link matching the given regular expression pattern.
The pattern is searched in the URL, link text, or name.
The pattern is searched in the URL and in the link text.
"""
return self.result.find_link(pattern) if self.result else None
return self._assert_result_for('links').find_link(pattern)
def follow_link(self, link):
def follow_link(self, link: Union[str, Link]) -> None:
"""Follow the given link."""

@@ -167,6 +202,8 @@ self._journey('follow_link', link)

@property
def headers(self):
def headers(self) -> CaseInsensitiveDict:
"""Return the request headers currently used by the browser."""
return self._session.headers
def reset_headers(self):
"""Reset the request headers currently used by the browser."""
self.headers.clear()

@@ -178,15 +215,20 @@ self.headers.update({

@property
def agent_string(self):
"""Get the agent string."""
def response_headers(self):
"""Get the headers returned with the current page."""
return self._assert_result_for('headers').headers
@property
def agent_string(self) -> Optional[str]:
"""Get the user agent string."""
return self.headers.get('User-Agent')
@agent_string.setter
def agent_string(self, agent):
"""Set the agent string to the given value."""
def agent_string(self, agent: str) -> None:
"""Set the user agent string to the given value."""
self.headers['User-Agent'] = agent
def showforms(self):
"""Pretty-print all of the forms.
def show_forms(self) -> None:
"""Pretty-print all forms on the page.
Include the global form (form elements outside of <form> pairs)
Include the global form (form elements outside <form> pairs)
as forms[0] if present.

@@ -197,4 +239,4 @@ """

def showlinks(self):
"""Pretty-print all of the links."""
def show_links(self) -> None:
"""Pretty-print all links on the page."""
info = log.info

@@ -210,3 +252,3 @@ links = self.links

def showhistory(self):
def show_history(self) -> None:
"""Pretty-print the history of links visited."""

@@ -224,37 +266,53 @@ info = log.info

@property
def links(self):
"""Return a list of all of the links on the page."""
return [] if self.result is None else self.result.links
def links(self) -> List[Link]:
"""Return a list of all links on the page."""
return self._assert_result_for('links').links
@property
def forms(self):
"""Return a list of all of the forms.
def history(self) -> List[ResultWrapper]:
"""Return a list of all pages visited by the browser."""
return self._history
Include the global form at index 0 if present.
@property
def forms(self) -> List[FormElement]:
"""Return a list of forms on the page.
This includes the global form at index 0 if present.
"""
return [] if self.result is None else self.result.forms
return self._assert_result_for('forms').forms
def form(self, formname=1):
def form(self, name: Union[str, int] = 1) -> Optional[FormElement]:
"""Return the first form that matches the given form name."""
return None if self.result is None else self.result.form(formname)
return self._assert_result_for('form').form(name)
def form_field(self, form, fieldname=1):
def form_field(self, form: FormElement = None,
name_or_num: Union[str, int] = 1) -> FieldElement:
"""Return the control that matches the given field name.
Must be a *unique* regex/exact string match.
Must be a *unique* regex/exact string match, but the returned
control can also be a CheckboxGroup or RadioGroup list.
Raises a TwillException if no such field or multiple fields are found.
"""
if form is None:
form = self._form
if form is None:
raise TwillException("Must specify a form for the field")
inputs = form.inputs
found_multiple = False
if isinstance(fieldname, str):
name = name_or_num if isinstance(name_or_num, str) else None
if fieldname in form.fields:
match_name = [c for c in inputs if c.name == fieldname]
if name:
if name in form.fields:
match_name = [c for c in inputs if c.name == name]
if len(match_name) > 1:
if all(hasattr(c, 'type') and c.type == 'checkbox'
for c in match_name):
return html.CheckboxGroup(match_name)
if all(hasattr(c, 'type') and c.type == 'radio'
for c in match_name):
return html.RadioGroup(match_name)
if all(getattr(c, 'type', None) == 'checkbox'
for c in match_name):
return CheckboxGroup(
cast(List[InputElement], match_name))
if all(getattr(c, 'type', None) == 'radio'
for c in match_name):
return RadioGroup(cast(List[InputElement], match_name))
else:

@@ -264,3 +322,3 @@ match_name = None

# test exact match to id
match_id = [c for c in inputs if c.get('id') == fieldname]
match_id = [c for c in inputs if c.get('id') == name]
if match_id:

@@ -277,12 +335,20 @@ if unique_match(match_id):

num = name_or_num if isinstance(name_or_num, int) else None
if num is None and name and name.isdigit():
try:
num = int(name)
except ValueError:
pass
# test field index
try:
return list(inputs)[int(fieldname) - 1]
except (IndexError, ValueError):
pass
if num is not None:
try:
return list(inputs)[num - 1]
except IndexError:
pass
if isinstance(fieldname, str):
if name:
# test regex match
regex = re.compile(fieldname)
regex = re.compile(name)
match_name = [c for c in inputs

@@ -296,3 +362,3 @@ if c.name and regex.search(c.name)]

# test field values
match_value = [c for c in inputs if c.value == fieldname]
match_value = [c for c in inputs if c.value == name]
if match_value:

@@ -305,9 +371,9 @@ if len(match_value) == 1:

if found_multiple:
raise TwillException(f'multiple matches to "{fieldname}"')
raise TwillException(f'no field matches "{fieldname}"')
raise TwillException(f'multiple matches to "{name_or_num}"')
raise TwillException(f'no field matches "{name_or_num}"')
def add_form_file(self, fieldname, fp):
self._form_files[fieldname] = fp
def add_form_file(self, field_name: str, fp: IO) -> None:
self._form_files[field_name] = fp
def clicked(self, form, control):
def clicked(self, form: FormElement, control: FieldElement) -> None:
"""Record a 'click' in a specific form."""

@@ -320,24 +386,22 @@ if self._form != form:

# record the last submit button clicked.
if hasattr(control, 'type') and control.type in ('submit', 'image'):
self.last_submit_button = control
if getattr(control, 'type', None) in ('submit', 'image'):
self.last_submit_button = cast(InputElement, control)
def submit(self, fieldname=None):
"""Submit the currently clicked form using the given field."""
if fieldname is not None:
fieldname = str(fieldname)
def submit(self, field_name: Optional[Union[str, int]] = None,
form_name: Optional[Union[str, int]] = None) -> None:
"""Submit the last or specified form using the given field."""
forms = self.forms
if not forms:
raise TwillException("no forms on this page!")
raise TwillException("There are no forms on this page.")
ctl = None
ctl: Optional[InputElement] = None
form = self._form
form = self._form if form_name is None else self.form(form_name)
if form is None:
if len(forms) == 1:
form = forms[0]
else:
if len(forms) > 1:
raise TwillException(
"more than one form;"
" you must select one (use 'fv') before submitting")
"There is more than one form on this page;"
" therefore you must specify a form explicitly"
" or select one (use 'fv') before submitting.")
form = forms[0]

@@ -348,16 +412,15 @@ action = form.action or ''

# no fieldname? see if we can use the last submit button clicked...
if fieldname is None:
if self.last_submit_button is None:
# no field name? see if we can use the last submit button clicked...
if field_name is None:
if form is not self._form or self.last_submit_button is None:
# get first submit button in form.
submits = [c for c in form.inputs
if hasattr(c, 'type') and
c.type in ('submit', 'image')]
if getattr(c, 'type', None) in ('submit', 'image')]
if submits:
ctl = submits[0]
ctl = cast(InputElement, submits[0])
else:
ctl = self.last_submit_button
else:
# fieldname given; find it
ctl = self.form_field(form, fieldname)
# field name given; find it
ctl = cast(InputElement, self.form_field(form, field_name))

@@ -382,5 +445,7 @@ # now set up the submission by building the request object that

payload = form.form_values()
if ctl is not None and ctl.get('name') is not None:
payload.append((ctl.get('name'), ctl.value))
payload = self._encode_payload(payload)
if ctl is not None:
name = ctl.get('name')
if name:
payload.append((name, ctl.value or ''))
encoded_payload = self._encode_payload(payload)

@@ -391,9 +456,10 @@ # now actually GO

r = self._session.post(
form.action, data=payload, headers=headers,
form.action, data=encoded_payload, headers=headers,
files=self._form_files)
else:
r = self._session.post(
form.action, data=payload, headers=headers)
form.action, data=encoded_payload, headers=headers)
else:
r = self._session.get(form.action, params=payload, headers=headers)
r = self._session.get(
form.action, params=encoded_payload, headers=headers)

@@ -403,6 +469,11 @@ self._form = None

self.last_submit_button = None
self._history.append(self.result)
if self.result is not None:
self._history.append(self.result)
self.result = ResultWrapper(r)
def save_cookies(self, filename):
def cookies(self) -> RequestsCookieJar:
"""Get all cookies from the current session."""
return self._session.cookies
def save_cookies(self, filename: str) -> None:
"""Save cookies into the given file."""

@@ -412,3 +483,3 @@ with open(filename, 'wb') as f:

def load_cookies(self, filename):
def load_cookies(self, filename: str) -> None:
"""Load cookies from the given file."""

@@ -418,8 +489,8 @@ with open(filename, 'rb') as f:

def clear_cookies(self):
"""Delete all of the cookies."""
def clear_cookies(self) -> None:
"""Delete all the cookies."""
self._session.cookies.clear()
def show_cookies(self):
"""Pretty-print all of the cookies."""
def show_cookies(self) -> None:
"""Pretty-print all the cookies."""
info = log.info

@@ -436,3 +507,3 @@ cookies = self._session.cookies

def decode(self, value):
def decode(self, value: Union[bytes, str]):
"""Decode a value using the current encoding."""

@@ -443,26 +514,25 @@ if isinstance(value, bytes) and self.encoding:

def xpath(self, path):
def xpath(self, path: str) -> List[HtmlElement]:
"""Evaluate an xpath expression."""
return self.result.xpath(path) if self.result else []
return self._assert_result_for('xpath').xpath(path)
def _encode_payload(self, payload):
"""Encode a payload with the current encoding."""
def _encode_payload(
self, payload: Sequence[Tuple[str, Union[str, bytes]]]
) -> List[Tuple[str, Union[str, bytes]]]:
"""Encode a payload with the current encoding if not utf-8."""
encoding = self.encoding
if not encoding or encoding.lower() in ('utf8', 'utf-8'):
return payload
new_payload = []
for name, val in payload:
if not isinstance(val, bytes):
val = val.encode(encoding)
new_payload.append((name, val))
return new_payload
return list(payload)
return [(name, val if isinstance(val, bytes) else val.encode(encoding))
for name, val in payload]
@staticmethod
def _get_meta_refresh(response):
def _get_meta_refresh(
response: Response) -> Tuple[Optional[int], Optional[str]]:
"""Get meta refresh interval and url from a response."""
try:
tree = html.fromstring(response.text)
tree = html_to_tree(response.text)
except ValueError:
# may happen when there is an XML encoding declaration
tree = html.fromstring(response.content)
tree = html_to_tree(response.content)
try:

@@ -490,3 +560,3 @@ content = tree.xpath( # "refresh" is case insensitive

def _journey(self, func_name, *args, **kwargs):
def _journey(self, func_name, *args, **_kwargs):
"""Execute the function with the given name and arguments.

@@ -526,2 +596,4 @@

raise TwillException
else:
raise TwillException(f"Unknown function {func_name!r}")

@@ -539,3 +611,3 @@ r = self._session.get(url, verify=self.verify)

# handle redirection via meta refresh (not handled in requests)
refresh_interval = _equiv_refresh_interval()
refresh_interval = get_equiv_refresh_interval()
if refresh_interval:

@@ -568,1 +640,3 @@ visited = set() # break circular refresh chains

browser = TwillBrowser() # the global twill browser instance
_disable_insecure_request_warnings() # should not warn for HTTP requests
+283
-240
twill/commands.py
"""
Implementation of all of the individual 'twill' commands available through
twill-sh.
Implementation of all the individual 'twill' commands available
through twill-sh.
"""

@@ -10,8 +10,8 @@

import sys
from typing import Any, Dict, Optional
from os.path import sep
import requests
from . import log, set_output, set_err_out, utils
from .agents import agents
from .browser import browser

@@ -21,2 +21,3 @@ from .errors import TwillException, TwillAssertionError

# noinspection SpellCheckingInspection
__all__ = [

@@ -29,12 +30,15 @@ 'add_auth', 'add_cleanup', 'add_extra_header', 'agent',

'find', 'follow',
'formaction', 'fa', 'formclear', 'formfile', 'formvalue', 'fv',
'getinput', 'getpassword',
'go', 'info', 'load_cookies', 'notfind', 'options',
'form_action', 'formaction', 'fa',
'form_clear', 'formclear', 'form_file', 'formfile',
'form_value', 'formvalue', 'fv',
'get_input', 'getinput', 'get_password', 'getpassword',
'go', 'info', 'load_cookies', 'not_find', 'notfind', 'options',
'redirect_error', 'redirect_output',
'reload', 'reset_browser', 'reset_error', 'reset_output',
'run', 'runfile', 'rf',
'run', 'run_file', 'runfile', 'rf',
'save_cookies', 'save_html',
'setglobal', 'setlocal',
'show', 'show_cookies', 'show_extra_headers',
'showforms', 'showhistory', 'showlinks',
'setglobal', 'set_global', 'setlocal', 'set_local',
'show', 'showcookies', 'show_cookies', 'show_extra_headers',
'showforms', 'show_forms', 'showhistory', 'show_history',
'showhtml', 'show_html', 'showlinks', 'show_links',
'sleep', 'submit',

@@ -54,3 +58,4 @@ 'tidy_ok', 'title', 'url']

def exit(code='0'):
# noinspection PyShadowingBuiltins
def exit(code: str = '0') -> None:
"""twill command: exit [<code>]

@@ -63,3 +68,3 @@

def go(url):
def go(url: str) -> None:
""">> go <url>

@@ -70,6 +75,5 @@

browser.go(url)
return browser.url
def reload():
def reload() -> None:
""">> reload

@@ -80,6 +84,5 @@

browser.reload()
return browser.url
def code(should_be):
def code(should_be: str) -> None:
""">> code <int>

@@ -89,12 +92,11 @@

"""
should_be = int(should_be)
if browser.code != should_be:
if browser.code != int(should_be):
raise TwillAssertionError(f"code is {browser.code} != {should_be}")
def tidy_ok():
def tidy_ok() -> None:
""">> tidy_ok
Assert that 'tidy' produces no warnings or errors when run on the current
page.
Assert that 'tidy' does not produce any warnings or errors when run on
the current page.

@@ -116,7 +118,7 @@ If 'tidy' cannot be run, will fail silently (unless 'require_tidy' option

def url(should_be):
""">> url <regex>
def url(should_be: str) -> str:
""">> url <pattern>
Check to make sure that the current URL matches the regex. The local
variable __match__ is set to the matching part of the URL.
Check to make sure that the current URL matches the regex pattern.
The local variable __match__ is set to the matching part of the URL.
"""

@@ -126,7 +128,7 @@ regex = re.compile(should_be)

m = None
if current_url is not None:
if current_url is None:
current_url = ''
m = None
else:
m = regex.search(current_url)
else:
current_url = ''

@@ -144,6 +146,7 @@ if not m:

def follow(what):
""">> follow <regex>
def follow(what: str) -> str:
""">> follow <pattern>
Find the first matching link on the page & visit it.
Find the first link on the page matching the given regex pattern and
then visit it.
"""

@@ -161,3 +164,3 @@ link = browser.find_link(what)

def _parse_find_flags(flags):
def _parse_find_flags(flags: str) -> int:
"""Helper function to parse the find flags."""

@@ -173,7 +176,7 @@ re_flags = 0

def find(what, flags=''):
""">> find <regex> [<flags>]
def find(what: str, flags='') -> str:
""">> find <pattern> [<flags>]
Succeed if the regular expression is on the page. Sets the local
variable __match__ to the matching text.
Succeed if the regular expression pattern can be found on the page.
Sets the local variable __match__ to the matching text.

@@ -187,3 +190,3 @@ Flags is a string consisting of the following characters:

For explanations of these, please see the Python re module
For explanations of regular expressions, please see the Python re module
documentation.

@@ -197,3 +200,3 @@ """

raise TwillAssertionError(f"no element to path '{what}'")
match_str = browser.decode(elements[0])
match_str = elements[0].text or ''
else:

@@ -205,8 +208,9 @@ match = re.search(what, page, flags=_parse_find_flags(flags))

local_dict['__match__'] = match_str
return match_str
def notfind(what, flags=''):
""">> notfind <regex> [<flags>]
def not_find(what: str, flags='') -> None:
""">> not_find <pattern> [<flags>]
Fail if the regular expression is on the page.
Fail if the regular expression pattern can be found on the page.
"""

@@ -221,3 +225,7 @@ try:

def back():
# noinspection SpellCheckingInspection
notfind = not_find # backward compatibility and convenience
def back() -> None:
""">> back

@@ -228,10 +236,29 @@

browser.back()
return browser.url
def show():
""">> show
def show(what: Optional[str] = None) -> None:
""">> show [<objects>]
Show the HTML for the current page.
Show the specified objects (html, cookies, forms, links, history).
"""
if not what:
what = 'html'
command = None
if what.isalpha():
command_name = f'show_{what}'
if command_name in __all__:
command = globals().get(command_name)
if not command:
raise TwillException(f'Cannot show "{what}".')
command()
def show_html() -> None:
""">> show_html
Show the HTML for the current page or show the specified objects
(which can be cookies, forms, history or links).
Note: Use browser.html to get the HTML programmatically.
"""
html = browser.html.strip()

@@ -241,6 +268,9 @@ log.info('')

log.info('')
return html
def echo(*strs):
# noinspection SpellCheckingInspection
showhtml = show_html # backward compatibility and consistency
def echo(*strs: str) -> None:
""">> echo <list> <of> <strings>

@@ -253,7 +283,7 @@

def save_html(filename=None):
def save_html(filename: Optional[str] = None) -> None:
""">> save_html [<filename>]
Save the HTML for the current page into <filename>. If no filename
given, construct the filename from the URL.
Save the HTML for the current page into <filename>.
If no filename given, construct the filename from the URL.
"""

@@ -271,3 +301,3 @@ html = browser.html

filename = 'index.html'
log.info("Using filename '%s'", filename)
log.info("Using filename '%s'.", filename)

@@ -285,3 +315,3 @@ encoding = browser.encoding or 'utf-8'

def sleep(interval=1):
def sleep(interval: str = "1") -> None:
""">> sleep [<interval>]

@@ -295,54 +325,3 @@

_agent_map = dict(
chrome40='Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36'
' (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36',
googlebot2='Mozilla/5.0 (compatible; Googlebot/2.1;'
' +http://www.google.com/bot.html)',
edge12='Mozilla/5.0 (Windows NT 10.0)'
' AppleWebKit/537.36 (KHTML, like Gecko)'
' Chrome/42.0.2311.135 Safari/537.36 Edge/12.10136',
firefox40='Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0)'
' Gecko/20100101 Firefox/40.1',
ie6='Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
ie7='Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
ie8='Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',
ie9='Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0)',
ie10='Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)',
ie11='Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko',
iemobile9='Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5;'
' Trident/5.0; IEMobile/9.0)',
opera7='Opera/7.0 (Windows NT 5.1; U) [en]',
opera8='Opera/8.00 (Windows NT 5.1; U; en)',
opera9='Opera/9.00 (Windows NT 5.2; U; en)',
opera10='Opera/9.80 (Windows NT 6.1; U; en) Presto/2.2.15 Version/10.00',
opera11='Opera/9.80 (Windows NT 6.1; U; en) Presto/2.7.62 Version/11.00',
opera12='Opera/12.0 (Windows NT 5.1; U; en) Presto/22.9.168 Version/12.00',
operamini7='Opera/9.80 (Android; Opera Mini/7.0.29952/28.2075; en)'
' Presto/2.8.119 Version/11.10',
operamini9='Opera/9.80 (J2ME/MIDP; Opera Mini/9 (Compatible; MSIE:9.0;'
' iPhone; BlackBerry9700; AppleWebKit/24.746; en)'
' Presto/2.5.25 Version/10.54',
konqueror3='Mozilla/5.0 (compatible; Konqueror/3.0; Linux)',
konqueror4='Mozilla/5.0 (compatible; Konqueror/4.0; Linux)'
' KHTML/4.0.3 (like Gecko)',
safari1='Mozilla/5.0 (Macintosh; PPC Mac OS X; en)'
' AppleWebKit/85.7 (KHTML, like Gecko) Safari/85.6',
safari2='Mozilla/5.0 (Macintosh; PPC Mac OS; en)'
' AppleWebKit/412 (KHTML, like Gecko) Safari/412',
safari3='Mozilla/5.0 (Macintosh; Intel Mac OS X; en)'
' AppleWebKit/522.7 (KHTML, like Gecko) Version/3.0 Safari/522.7',
safari4='Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_6; en)'
' AppleWebKit/530.9+ (KHTML, like Gecko)'
'Version/4.0 Safari/528.16',
safari5='Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en)'
' AppleWebKit/534.1+ (KHTML, like Gecko)'
' Version/5.0 Safari/533.16',
safari6='Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536'
'.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25',
safari7='Mozilla/5.0 (iPad; CPU OS 7_1_2 like Mac OS X) AppleWebKit/537'
'.51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D257 Safari/9537.53',
)
def agent(what):
def agent(what: str) -> None:
""">> agent <agent>

@@ -353,66 +332,86 @@

Some convenient shortcuts:
ie5, ie55, ie6, moz17, opera7, konq32, saf11, aol9.
chrome_107, firefox_106, safari_605, edge_107, ie_11.
See twill.agents for a list of all available shortcuts.
"""
what = what.strip()
agent = _agent_map.get(what, what)
agent = agents.get(what, what)
browser.agent_string = agent
def submit(submit_button=None):
""">> submit [<buttonspec>]
def submit(submit_button: Optional[str] = None,
form_name: Optional[str] = None) -> None:
""">> submit [<submit_button> [<form_name>]]
Submit the current form (the one last clicked on) by clicking on the
n'th submission button. If no "buttonspec" is given, submit the current
form by using the last clicked submit button.
given submission button. If no 'submit_button' is given, submit the
current form by using the last clicked submit button.
The form to submit is the last form clicked on with a 'formvalue' command.
The form to submit is the last form clicked on with a 'form_value' command
unless explicitly specified given the
The button used to submit is chosen based on 'buttonspec'. If 'buttonspec'
is given, it's matched against buttons using the same rules that
'formvalue' uses. If 'buttonspec' is not given, submit uses the last
submit button clicked on by 'formvalue'. If none can be found,
submit submits the form with no submit button clicked.
The button used to submit is chosen based on 'submit_button'.
If 'submit_button' is given, it's matched against buttons using
the same rules that 'form_value' uses. If 'button_name' is not given,
this function uses the last submit button clicked on by 'form_value'.
If none can be found, it submits the form with no submit button clicked.
"""
browser.submit(submit_button)
browser.submit(submit_button, form_name)
def showforms():
""">> showforms
def show_forms() -> None:
""">> show_forms
Show all of the forms on the current page.
Show all the forms on the current page.
Note: Use browser.forms to get the forms programmatically.
"""
browser.showforms()
return browser.forms
browser.show_forms()
def showlinks():
""">> showlinks
# noinspection SpellCheckingInspection
showforms = show_forms # backward compatibility and convenience
Show all of the links on the current page.
def show_links() -> None:
""">> show_links
Show all the links on the current page.
Note: Use browser.links to get the links programmatically.
"""
browser.showlinks()
return browser.links
browser.show_links()
def showhistory():
""">> showhistory
# noinspection SpellCheckingInspection
showlinks = show_links # backward compatibility and convenience
def show_history() -> None:
""">> show_history
Show the browser history (what URLs were visited).
Note: Use browser.history to get the history programmatically.
"""
browser.showhistory()
return browser._history
browser.show_history()
def formclear(formname):
""">> formclear <formname>
# noinspection SpellCheckingInspection
showhistory = show_history # backward compatibility and convenience
Run 'clear' on all of the controls in this form.
def form_clear(form_name: str) -> None:
""">> form_clear <form_name>
Run 'clear' on all the controls in this form.
"""
form = browser.form(formname)
form = browser.form(form_name)
if form is None:
raise TwillAssertionError("Form not found")
for control in form.inputs:
if 'readonly' in control.attrib or 'disabled' in control.attrib or (
hasattr(control, 'type') and
control.type in ('submit', 'image', 'hidden')):
continue
else:
if not ('readonly' in control.attrib
or 'disabled' in control.attrib
or getattr(control, 'type', None)
in ('submit', 'image', 'hidden')):
del control.value

@@ -422,46 +421,51 @@ browser.last_submit_button = None

def formvalue(formname, fieldname, value):
""">> formvalue <formname> <field> <value>
# noinspection SpellCheckingInspection
formclear = form_clear # backward compatibility and convenience
def form_value(form_name: str, field_name: str, value: str) -> None:
""">> form_value <form_name> <field_name> <value>
Set value of a form field.
There are some ambiguities in the way 'formvalue' deals with lists:
'formvalue' will *add* the given value to a list of multiple selection,
There are some ambiguities in the way 'form_value' deals with lists:
'form_value' will *add* the given value to a list of multiple selection,
for lists that allow it.
Forms are matched against 'formname' as follows:
Forms are matched against 'form_name' as follows:
1. regex match to actual form name;
2. if 'formname' is an integer, it's tried as an index.
2. if 'form_name' is an integer, it's tried as an index.
Form controls are matched against 'fieldname' as follows:
Form controls are matched against 'field_name' as follows:
1. unique exact match to control name;
2. unique regex match to control name;
3. if fieldname is an integer, it's tried as an index;
3. if field_name is an integer, it's tried as an index;
4. unique & exact match to submit-button values.
'formvalue' ignores read-only fields completely; if they're readonly,
'form_value' ignores read-only fields completely; if they're readonly,
nothing is done, unless the config options ('config' command) are
changed.
'formvalue' is available as 'fv' as well.
'form_value' is available as 'fv' as well.
"""
form = browser.form(formname)
form = browser.form(form_name)
if form is None:
raise TwillAssertionError("no matching forms!")
raise TwillAssertionError("Form not found")
control = browser.form_field(form, fieldname)
control = browser.form_field(form, field_name)
browser.clicked(form, control)
if hasattr(control, 'attrib') and 'readonly' in control.attrib:
attrib = getattr(control, 'attrib', {})
if 'readonly' in attrib:
if options['readonly_controls_writeable']:
log.info('forcing read-only form field to writeable')
del control.attrib['readonly']
log.info('Forcing read-only form field to writeable.')
del attrib['readonly']
else:
log.info('form field is read-only or ignorable; nothing done.')
log.info('Form field is read-only or ignorable; nothing done.')
return
if hasattr(control, 'type') and control.type == 'file':
if getattr(control, 'type', None) == 'file':
raise TwillException(
'form field is for file upload; use "formfile" instead')
'form field is for file upload; use "form_file" instead')

@@ -472,22 +476,27 @@ value = browser.decode(value)

fv = formvalue # alias
# noinspection SpellCheckingInspection
fv = formvalue = form_value # backward compatibility and convenience
def formaction(formname, action):
""">> formaction <formname> <action_url>
def form_action(form_name: str, action_url: str) -> None:
""">> form_action <form_name> <action_url>
Sets action parameter on form to action_url.
'formaction' is available as 'fa' as well.
'form_action' is available as 'fa' as well.
"""
form = browser.form(formname)
log.info("Setting action for form %s to %s", form, action)
form.action = action
form = browser.form(form_name)
if form is None:
raise TwillAssertionError("Form not found")
log.info("Setting action for form %s to %s.", form, action_url)
form.action = action_url
fa = formaction # alias
# noinspection SpellCheckingInspection
fa = formaction = form_action # backward compatibility and convenience
def formfile(formname, fieldname, filename, content_type=None):
""">> formfile <form> <field> <filename> [<content_type>]
def form_file(form_name: str, field_name: str, filename: str,
content_type: Optional[str] = None) -> None:
""">> form_file <form_name> <field_name> <filename> [<content_type>]

@@ -498,6 +507,8 @@ Upload a file via an "upload file" form field.

form = browser.form(formname)
control = browser.form_field(form, fieldname)
form = browser.form(form_name)
if form is None:
raise TwillAssertionError("Form not found")
control = browser.form_field(form, field_name)
if not (hasattr(control, 'type') and control.type == 'file'):
if getattr(control, 'type', None) != 'file':
raise TwillException('ERROR: field is not a file upload field!')

@@ -508,11 +519,15 @@

fp = open(filename, 'r' if plain else 'rb')
browser.add_form_file(fieldname, fp)
browser.add_form_file(field_name, fp)
log.info(
'Added file "%s" to file upload field "%s"', filename, control.name)
'Added file "%s" to file upload field "%s".', filename, field_name)
def extend_with(module_name):
""">> extend_with <module>
# noinspection SpellCheckingInspection
formfile = form_file # backward compatibility and convenience
def extend_with(module_name: str) -> None:
""">> extend_with <module_name>
Import contents of given module.

@@ -527,11 +542,11 @@ """

mod = sys.modules.get(module_name)
mod = sys.modules[module_name]
from . import parse, shell
fnlist = getattr(mod, '__all__', None)
if fnlist is None:
fnlist = [fn for fn in dir(mod) if callable(getattr(mod, fn))]
fn_list = getattr(mod, '__all__', None)
if fn_list is None:
fn_list = [fn for fn in dir(mod) if callable(getattr(mod, fn))]
for command in fnlist:
for command in fn_list:
fn = getattr(mod, command)

@@ -549,5 +564,5 @@ shell.add_command(command, fn.__doc__)

else:
if fnlist:
if fn_list:
info('New commands:\n')
for name in fnlist:
for name in fn_list:
info('\t%s', name)

@@ -557,4 +572,4 @@ info('')

def getinput(prompt):
""">> getinput <prompt>
def get_input(prompt: str) -> str:
""">> get_input <prompt>

@@ -571,5 +586,9 @@ Get input, store it in '__input__'.

def getpassword(prompt):
""">> getpassword <prompt>
# noinspection SpellCheckingInspection
getinput = get_input # backward compatibility and convenience
def get_password(prompt: str) -> str:
""">> get_password <prompt>
Get a password ("invisible input"), store it in '__password__'.

@@ -587,6 +606,10 @@ """

def save_cookies(filename):
# noinspection SpellCheckingInspection
getpassword = get_password # backward compatibility and convenience
def save_cookies(filename: str) -> None:
""">> save_cookies <filename>
Save all of the current cookies to the given file.
Save all the current cookies to the given file.
"""

@@ -596,3 +619,3 @@ browser.save_cookies(filename)

def load_cookies(filename):
def load_cookies(filename: str) -> None:
""">> load_cookies <filename>

@@ -605,3 +628,3 @@

def clear_cookies():
def clear_cookies() -> None:
""">> clear_cookies

@@ -614,6 +637,8 @@

def show_cookies():
def show_cookies() -> None:
""">> show_cookies
Show all of the cookies in the cookie jar.
Show all the cookies in the cookie jar.
Note: Use browser.cookies to get the cookies programmatically.
"""

@@ -623,3 +648,7 @@ browser.show_cookies()

def add_auth(realm, uri, user, passwd):
# noinspection SpellCheckingInspection
showcookies = show_cookies # backward compatibility and convenience
def add_auth(realm: str, uri: str, user: str, passwd: str) -> None:
""">> add_auth <realm> <uri> <user> <passwd>

@@ -629,12 +658,14 @@

"""
browser.creds = ((uri, realm), (user, passwd))
if realm is not None:
browser.add_creds((uri, realm), user, passwd)
log.info(
"Added auth info: realm '%s' / URI '%s' / user '%s'.",
realm, uri, user)
if realm is None or options['with_default_realm']:
browser.add_creds(uri, user, passwd)
if realm is None:
log.info("Added auth info: URI '%s' / user '%s'.", uri, user)
log.info(
"Added auth info: realm '%s' / URI '%s' / user '%s'", realm, uri, user)
if options['with_default_realm']:
browser.creds = (uri, (user, passwd))
def debug(what, level):
def debug(what: str, level: str) -> None:
""">> debug <what> <level>

@@ -650,19 +681,19 @@

try:
level = int(level)
num_level = int(level)
except ValueError:
level = 1 if utils.make_boolean(level) else 0
num_level = 1 if utils.make_boolean(level) else 0
log.info('DEBUG: setting %s debugging to level %d', what, level)
log.info('DEBUG: Setting %s debugging to level %d.', what, num_level)
if what == 'http':
requests.packages.urllib3.connectionpool.debuglevel = level
browser.debug_level = num_level
elif what == 'equiv-refresh':
browser.show_refresh = level > 0
browser.show_refresh = num_level > 0
elif what == 'commands':
parse.log_commands(level > 0)
parse.log_commands(num_level > 0)
else:
raise TwillException(f'unknown debugging type: "{what}"')
raise TwillException(f'Unknown debugging type: "{what}"')
def run(cmd):
def run(cmd: str) -> None:
""">> run <command>

@@ -684,8 +715,8 @@

def runfile(*args):
""">> runfile <file1> [<file2> ...]
def run_file(*args: str) -> None:
""">> run_file <file1> [<file2> ...]
Execute the given twill scripts or directories of twill scripts.
'runfile' is available as 'rf' as well.
'run_file' is available as 'rf' as well.
"""

@@ -699,6 +730,7 @@ from . import parse

rf = runfile # alias
# noinspection SpellCheckingInspection
rf = runfile = run_file # backward compatibility and convenience
def add_cleanup(*args):
def add_cleanup(*args: str) -> None:
""">> add_cleanup <file1> [<file2> ...]

@@ -716,4 +748,4 @@

def setglobal(name, value):
"""setglobal <name> <value>
def set_global(name: str, value: str) -> None:
"""set_global <name> <value>

@@ -726,5 +758,9 @@ Sets the variable <name> to the value <value> in the global namespace.

def setlocal(name, value):
"""setlocal <name> <value>
# noinspection SpellCheckingInspection
setglobal = set_global # backward compatibility and convenience
def set_local(name: str, value: str) -> None:
"""set_local <name> <value>
Sets the variable <name> to the value <value> in the local namespace.

@@ -736,6 +772,10 @@ """

def title(what):
""">> title <regex>
# noinspection SpellCheckingInspection
setlocal = set_local # backward compatibility and convenience
Succeed if the regular expression is in the page title.
def title(what: str) -> str:
""">> title <pattern>
Succeed if the regular expression pattern is in the page title.
"""

@@ -745,7 +785,10 @@ regex = re.compile(what)

log.info("title is '%s'", title)
if title is None:
log.info("The page has no title.")
else:
log.info("The title is '%s'.", title)
m = regex.search(title)
if not m:
raise TwillAssertionError(f"title does not contain '{what}'")
m = regex.search(title) if title else None
if m is None:
raise TwillAssertionError(f"The title does not contain '{what}'.")

@@ -762,3 +805,3 @@ if m.groups():

def redirect_output(filename):
def redirect_output(filename: str) -> None:
""">> redirect_output <filename>

@@ -772,3 +815,3 @@

def reset_output():
def reset_output() -> None:
""">> reset_output

@@ -781,3 +824,3 @@

def redirect_error(filename):
def redirect_error(filename: str) -> None:
""">> redirect_error <filename>

@@ -791,3 +834,3 @@

def reset_error():
def reset_error() -> None:
""">> reset_error

@@ -800,3 +843,3 @@

def add_extra_header(header_key, header_value):
def add_extra_header(header_key: str, header_value: str) -> None:
""">> add_header <name> <value>

@@ -810,3 +853,3 @@

def show_extra_headers():
def show_extra_headers() -> None:
""">> show_extra_headers

@@ -827,3 +870,3 @@

def clear_extra_headers():
def clear_extra_headers() -> None:
""">> clear_extra_headers

@@ -837,3 +880,3 @@

default_options = dict(
default_options: Dict[str, Any] = dict(
equiv_refresh_interval=2,

@@ -847,3 +890,3 @@ readonly_controls_writeable=False,

def config(key=None, value=None):
def config(key: Optional[str] = None, value: Any = None) -> None:
""">> config [<key> [<int value>]]

@@ -886,3 +929,3 @@

def info():
def info() -> None:
""">> info

@@ -897,3 +940,3 @@

content_type = browser.result.headers['content-type']
content_type = browser.response_headers['content-type']
is_html = content_type and content_type.split(';', 1)[0] == 'text/html'

@@ -900,0 +943,0 @@ code = browser.code

+5
-5
twill/extensions/argparse.py

@@ -25,14 +25,14 @@ """

if len(shell.twillargs) < require:
if len(shell.twill_args) < require:
from twill.errors import TwillAssertionError
given = len(shell.twillargs)
given = len(shell.twill_args)
raise TwillAssertionError(
f"too few arguments; {given} rather than {require}")
if shell.twillargs:
for n, arg in enumerate(shell.twillargs, 1):
if shell.twill_args:
for n, arg in enumerate(shell.twill_args, 1):
global_dict[f"arg{n}"] = arg
n = len(shell.twillargs)
n = len(shell.twill_args)
log.info("get_args: loaded %d args as $arg1..$arg%d.", n, n)
else:
log.info("no arguments to parse!")
+13
-10
twill/extensions/check_links.py

@@ -8,3 +8,3 @@ """

Make sure that all of the HTTP links on the current page can be visited
Make sure that all the HTTP links on the current page can be visited
successfully. If 'pattern' is given, check only URLs that match that

@@ -15,3 +15,3 @@ regular expression.

links are silently collected across all calls to check_links. The
function 'report_bad_links' can then be used to report all of the links,
function 'report_bad_links' can then be used to report all the links,
together with their referring pages.

@@ -22,2 +22,4 @@ """

from typing import Dict, List, Set
from twill import browser, commands, log, utils

@@ -32,4 +34,4 @@ from twill.errors import TwillAssertionError

good_urls = set()
bad_urls = dict()
good_urls: Set[str] = set()
bad_urls: Dict[str, Set[str]] = dict()

@@ -40,3 +42,3 @@

Make sure that all of the HTTP links on the current page can be visited
Make sure that all the HTTP links on the current page can be visited
with an HTTP response 200 (success). If 'pattern' is given, interpret

@@ -46,3 +48,3 @@ it as a regular expression that link URLs must contain in order to be

check_links http://.*\\.google\\.com
check_links https://.*\\.google\\.com

@@ -65,3 +67,3 @@ would check only links to google URLs. Note that because 'follow'

collected_urls = set()
collected_urls: Set[str] = set()

@@ -77,2 +79,3 @@ links = browser.links

# noinspection HttpUrlsUsage
if not url.startswith(('http://', 'https://')):

@@ -94,3 +97,3 @@ debug("url '%s' is not an HTTP link; ignoring", url)

failed = []
failed: List[str] = []
for url in sorted(collected_urls):

@@ -119,3 +122,3 @@ debug("Checking %s", url)

for url in failed:
referrers = bad_urls.getdefault(url, set())
referrers = bad_urls.setdefault(url, set())
info('*** %s', browser.url)

@@ -135,3 +138,3 @@ referrers.add(browser.url)

Report all of the links collected across check_links runs (collected
Report all the links collected across check_links runs (collected
if and only if the config option check_links.only_collect_bad_links

@@ -138,0 +141,0 @@ is set).

+13
-13
twill/extensions/dns_check.py

@@ -35,3 +35,3 @@ """

for answer in _query(host, 'A', server):
for answer in _resolve(host, 'A', server):
if ipaddress == answer.address:

@@ -55,3 +55,3 @@ return True

for answer in _query(host, 'CNAME', server):
for answer in _resolve(host, 'CNAME', server):
if cname == answer.target:

@@ -73,3 +73,3 @@ return True

for answer in _query(host, 1, server):
for answer in _resolve(host, 1, server):
if ipaddress == answer.address:

@@ -88,3 +88,3 @@ return True

for rdata in _query(host, 'MX', server):
for rdata in _resolve(host, 'MX', server):
if mailserver == rdata.exchange:

@@ -103,3 +103,3 @@ return True

for answer in _query(host, 'NS', server):
for answer in _resolve(host, 'NS', server):
if query_ns == answer.target:

@@ -125,7 +125,7 @@ return True

r = Resolver()
resolver = Resolver()
if server:
r.nameservers = [_resolve_name(server, None)]
resolver.nameservers = [_resolve_name(server, None)]
answers = r.query(name)
answers = resolver.resolve(name)

@@ -135,8 +135,8 @@ return str(answers[0])

def _query(query, query_type, server):
"""Query, perhaps via the given name server (None to use default)."""
r = Resolver()
def _resolve(query, query_type, server):
"""Resolve, perhaps via the given name server (None to use default)."""
resolver = Resolver()
if server:
r.nameservers = [_resolve_name(server, None)]
resolver.nameservers = [_resolve_name(server, None)]
return r.query(query, query_type)
return resolver.resolve(query, query_type)
+24
-24
twill/extensions/formfill.py

@@ -9,9 +9,9 @@ """

* fv_match -- fill in *all* fields that match a regex (unlike 'formvalue'
* fv_match -- fill in *all* fields that match a regex (unlike 'form_value'
which will complain about multiple matches). Useful for forms
with lots of repeated fieldnames -- 'field-1', 'field-2', etc.
with lots of repeated field names -- 'field-1', 'field-2', etc.
* fv_multi -- fill in multiple form fields at once, e.g.
fv_multi <formname> field1=value1 field2=value2 field3=value3
fv_multi <form_name> field1=value1 field2=value2 field3=value3

@@ -28,9 +28,9 @@ * fv_multi_sub -- same as 'fv_multi', followed by a 'submit'.

def fv_match(form_name, regex, value):
""">> fv_match <formname> <field regex> <value>
def fv_match(form_name: str, field_pattern: str, value: str) -> None:
""">> fv_match <form_name> <field_pattern> <value>
Set value of *all* form fields with a name that matches the given
regular expression.
regular expression pattern.
(Unlike 'formvalue' or 'fv', this will not complain about multiple
(Unlike 'form_value' or 'fv', this will not complain about multiple
matches!)

@@ -43,3 +43,3 @@ """

regex = re.compile(regex)
regex = re.compile(field_pattern)

@@ -64,8 +64,8 @@ matches = [ctl for ctl in form.inputs

def fv_multi_match(form_name, regex, *values):
""">> fv_multi_match <formname> <field regex> <value> [<value> [<value>..]]
def fv_multi_match(form_name: str, field_pattern: str, *values: str) -> None:
""">> fv_multi_match <form_name> <field_pattern> <value>...
Set value of each consecutive matching form field with the next specified
value. If there are no more values, use the last for all remaining form
fields
Set value of each consecutive form field matching the given pattern with
the next specified value. If there are no more values, use the last for
all remaining form fields.
"""

@@ -77,3 +77,3 @@ form = browser.form(form_name)

regex = re.compile(regex)
regex = re.compile(field_pattern)

@@ -98,27 +98,27 @@ matches = [

def fv_multi(form_name, *pairs):
""">> fv_multi <formname> [<pair1> [<pair2> [<pair3>]]]
def fv_multi(form_name: str, *pairs: str) -> None:
""">> fv_multi <form_name> <pair>...
Set multiple form fields; each pair should be of the form
fieldname=value
field_name=value
The pair will be split around the first '=', and
'fv <formname> fieldname value' will be executed in the order the
'fv <form_name> field_name value' will be executed in the order the
pairs are given.
"""
for p in pairs:
field_name, value = p.split('=', 1)
for pair in pairs:
field_name, value = pair.split('=', 1)
commands.fv(form_name, field_name, value)
def fv_multi_sub(form_name, *pairs):
""">> fv_multi_sub <formname> [<pair1> [<pair2> [<pair3>]]]
def fv_multi_sub(form_name: str, *pairs: str) -> None:
""">> fv_multi_sub <form_name> <pair>...
Set multiple form fields (as with 'fv_multi') and then submit().
"""
for p in pairs:
field_name, value = p.split('=', 1)
for pair in pairs:
field_name, value = pair.split('=', 1)
commands.fv(form_name, field_name, value)
commands.submit()
+2
-2
twill/extensions/mailman_sf.py

@@ -33,6 +33,6 @@ """

"""
_formvalue_by_regex_setall('1', '^\\d+$', '3')
_form_value_by_regex_setall('1', '^\\d+$', '3')
def _formvalue_by_regex_setall(form_name, field_name, value):
def _form_value_by_regex_setall(form_name, field_name, value):
form = browser.form(form_name)

@@ -39,0 +39,0 @@ if not form:

+6
-4
twill/extensions/require.py

@@ -49,2 +49,3 @@ """

# install the post-load hook function.
# noinspection PyProtectedMember
hooks = browser._post_load_hooks

@@ -66,2 +67,3 @@ if _require_post_load_hook not in hooks:

"""
# noinspection PyProtectedMember
hooks = browser._post_load_hooks

@@ -80,7 +82,7 @@ hooks = [fn for fn in hooks if fn != _require_post_load_hook]

"""
from check_links import good_urls
from .check_links import good_urls # type: ignore
good_urls.clear()
def _require_post_load_hook(action, *args, **kwargs):
def _require_post_load_hook(action, *_args, **_kwargs):
"""Post load hook function to be called after each page is loaded.

@@ -104,6 +106,6 @@

log.debug('REQUIRING success')
commands.code(200)
commands.code("200")
elif what == 'links_ok':
from check_links import check_links, good_urls
from check_links import check_links, good_urls # type: ignore

@@ -110,0 +112,0 @@ ignore_always = True

+0
-12
twill/fork.py
"""twill multiprocess execution system."""
# This file is part of the twill source distribution.
#
# twill is a extensible scriptlet language for testing Web apps,
# available at http://twill.idyll.org/.
#
# Contact author: C. Titus Brown, titus@idyll.org.
#
# This program and all associated source code files are Copyright (C)
# 2005-2007 by C. Titus Brown. It is released under the MIT license;
# please see the included LICENSE.txt file for more information, or
# go to http://www.opensource.org/licenses/mit-license.php.
import sys

@@ -16,0 +4,0 @@ import os

+2
-1
twill/namespaces.py

@@ -10,4 +10,5 @@ """Global and local dictionaries, and initialization/utility functions."""

This must be done after all the other modules are loaded, so that all
of the commands are already defined.
the commands are already defined.
"""
# noinspection PyCompatibility
from . import commands, parse

@@ -14,0 +15,0 @@ 

+17
-14
twill/parse.py

@@ -7,2 +7,3 @@ """Code parsing and evaluation for the twill mini-language."""

from io import StringIO
from typing import List

@@ -13,3 +14,5 @@ from pyparsing import (

from . import browser, commands, log, namespaces
# noinspection PyCompatibility
from . import commands, log, namespaces
from .browser import browser
from .errors import TwillNameError

@@ -25,4 +28,4 @@

# basically, a valid Python identifier:
command = Word(alphas + '_', alphanums + '_')
command = command.setResultsName('command')
command_word = Word(alphas + '_', alphanums + '_')
command = command_word.setResultsName('command')
command.setName('command')

@@ -54,4 +57,4 @@

arguments = Group(ZeroOrMore(quotedArg | plainArg))
arguments = arguments.setResultsName('arguments')
arguments_group = Group(ZeroOrMore(quotedArg | plainArg))
arguments = arguments_group.setResultsName('arguments')
arguments.setName('arguments')

@@ -68,3 +71,3 @@

command_list = [] # filled in by namespaces.init_global_dict().
command_list: List[str] = [] # filled in by namespaces.init_global_dict().

@@ -80,3 +83,3 @@

"""
newargs = []
new_args: List[str] = []
for arg in args:

@@ -93,5 +96,5 @@ # __variable substitution

if isinstance(val, str):
newargs.append(val)
new_args.append(val)
else:
newargs.extend(val)
new_args.extend(val)

@@ -104,9 +107,9 @@ # $variable substitution

val = arg
newargs.append(val)
new_args.append(val)
else:
newargs.append(variable_substitution(
new_args.append(variable_substitution(
arg, globals_dict, locals_dict))
newargs = [arg.replace('\\n', '\n') for arg in newargs]
return newargs
new_args = [arg.replace('\\n', '\n') for arg in new_args]
return new_args

@@ -142,3 +145,3 @@

_log_commands = log.debug
_log_commands = log.debug # type: ignore

@@ -145,0 +148,0 @@ 

+77
-61
twill/shell.py

@@ -13,15 +13,20 @@ """

from cmd import Cmd
from io import TextIOWrapper
from optparse import OptionParser
from typing import Any, Callable, List, Optional
try:
import readline
from readline import read_history_file, write_history_file # type: ignore
except ImportError:
readline = None
read_history_file = write_history_file = None # type: ignore
# noinspection PyCompatibility
from . import (
browser, commands, execute_file,
commands, execute_file,
log, log_levels, set_log_level, set_output,
namespaces, parse, shutdown, __url__, __version__)
from .browser import browser
from .utils import gather_filenames, Singleton
__all__ = ['main']
python_version = sys.version.split(None, 1)[0]

@@ -37,3 +42,3 @@

def make_cmd_fn(cmd):
def make_cmd_fn(cmd: str) -> Callable[[str], None]:
"""Make a command function.

@@ -46,3 +51,3 @@

def do_cmd(rest_of_line, cmd=cmd):
def do_cmd(rest_of_line: str, cmd: str = cmd) -> None:
global_dict, local_dict = namespaces.get_twill_glocals()

@@ -71,3 +76,3 @@

def make_help_cmd(cmd, docstring):
def make_help_cmd(cmd: str, docstring: str) -> Callable[[str], None]:
"""Make a help command function.

@@ -78,3 +83,3 @@

"""
def help_cmd(message=docstring, cmd=cmd):
def help_cmd(message: str = docstring, cmd: str = cmd) -> None:
message = message.strip()

@@ -95,3 +100,3 @@ width = 7 + len(cmd)

def add_command(cmd, docstring):
def add_command(cmd: str, docstring: str) -> None:
"""Add a command with given docstring to the shell."""

@@ -103,7 +108,2 @@ shell = get_command_shell()

def get_command_shell():
"""Get the command shell."""
return getattr(TwillCommandLoop, '__it__', None)
class TwillCommandLoop(Singleton, Cmd):

@@ -119,3 +119,6 @@ """The command-line interpreter for twill commands.

def __init__(self, stdin=None, initial_url=None, fail_on_unknown=False):
def __init__(
self, stdin: Optional[TextIOWrapper] = None,
initial_url: Optional[str] = None,
fail_on_unknown: bool = False) -> None:
Cmd.__init__(self, stdin=stdin)

@@ -128,6 +131,6 @@

# import readline history, if available.
if readline:
# import readline history, if available/possible.
if read_history_file:
try:
readline.read_history_file('.twill-history')
read_history_file('.twill-history')
except IOError:

@@ -145,3 +148,3 @@ pass

self.names = []
self.names: List[str] = []

@@ -155,3 +158,3 @@ global_dict, local_dict = namespaces.get_twill_glocals()

def add_command(self, command, docstring):
def add_command(self, command: str, docstring: str) -> None:
"""Add the given command into the lexicon of all commands."""

@@ -169,12 +172,13 @@ do_name = f'do_{command}'

def get_names(self):
def get_names(self) -> List[str]:
"""Return the list of commands."""
return self.names
def complete_formvalue(self, text, line, begin, end):
"""Command arg completion for the formvalue command.
def complete_form_value(
self, text: str, line: str, _begin: int, _end: int) -> List[str]:
"""Command arg completion for the form_value command.
The twill command has the following syntax:
formvalue <formname> <field> <value>
form_value <form_name> <field_name> <value>
"""

@@ -184,11 +188,12 @@ cmd, args = parse.parse_command(line + '.', {}, {})

if place == 1:
return self.provide_formname(text)
elif place == 2:
formname = args[0]
return self.provide_field(formname, text)
return self.provide_form_name(text)
if place == 2:
form_name = args[0]
return self.provide_field_name(form_name, text)
return []
complete_fv = complete_formvalue # alias
complete_fv = complete_form_value # alias
def provide_formname(self, prefix):
@staticmethod
def provide_form_name(prefix: str) -> List[str]:
"""Provide the list of form names on the given page."""

@@ -198,5 +203,5 @@ names = []

for form in forms:
id = form.attrib.get('id')
if id and id.startswith(prefix):
names.append(id)
form_id = form.attrib.get('id')
if form_id and form_id.startswith(prefix):
names.append(form_id)
continue

@@ -208,11 +213,12 @@ name = form.attrib.get('name')

def provide_field(self, formname, prefix):
"""Provide the list of fields for the given formname or number."""
@staticmethod
def provide_field_name(form_name: str, prefix: str) -> List[str]:
"""Provide the list of fields for the given form_name or number."""
names = []
form = browser.form(formname)
form = browser.form(form_name)
if form is not None:
for field in form.inputs:
id = field.attrib.get('id')
if id and id.startswith(prefix):
names.append(id)
field_id = field.attrib.get('id')
if field_id and field_id.startswith(prefix):
names.append(field_id)
continue

@@ -224,3 +230,3 @@ name = field.name

def _set_prompt(self):
def _set_prompt(self) -> None:
""""Set the prompt to the current page."""

@@ -232,7 +238,7 @@ url = browser.url

def precmd(self, line):
def precmd(self, line: str) -> str:
"""Run before each command; save."""
return line
def postcmd(self, stop, line):
def postcmd(self, stop: bool, line: str) -> bool:
""""Run after each command; set prompt."""

@@ -242,3 +248,3 @@ self._set_prompt()

def default(self, line):
def default(self, line: str) -> None:
""""Called when an unknown command is executed."""

@@ -268,29 +274,34 @@

def emptyline(self):
def emptyline(self) -> Any:
"""Handle empty lines (by ignoring them)."""
pass
def do_EOF(self, *args):
@staticmethod
def do_EOF(*_args: str) -> None:
"""Exit on CTRL-D"""
if readline:
readline.write_history_file('.twill-history')
if write_history_file:
write_history_file('.twill-history')
raise SystemExit()
def help_help(self):
@staticmethod
def help_help() -> None:
"""Show help for the help command."""
log.info("\nWhat do YOU think the command 'help' does?!?\n")
def do_version(self, *args):
@staticmethod
def do_version(*_args: str) -> None:
"""Show the version number of twill."""
log.info(version_info)
def help_version(self):
@staticmethod
def help_version() -> None:
"""Show help for the version command."""
log.info("\nPrint version information.\n")
def do_exit(self, *args):
def do_exit(self, *_args: str) -> None:
"""Exit the twill shell."""
raise SystemExit()
def help_exit(self):
@staticmethod
def help_exit() -> None:
"""Show help for the exit command."""

@@ -303,3 +314,8 @@ log.info("\nExit twill.\n")

twillargs = [] # contains sys.argv *after* last '--'
def get_command_shell() -> Optional[TwillCommandLoop]:
"""Get the command shell."""
return getattr(TwillCommandLoop, '__it__', None)
twill_args: List[str] = [] # contains sys.argv *after* last '--'
interactive = False # 'True' if interacting with user

@@ -309,3 +325,3 @@

def main():
global twillargs, interactive
global twill_args, interactive

@@ -345,11 +361,11 @@ # show the shorthand name for usage

# parse arguments
sysargs = sys.argv[1:]
if '--' in sysargs:
for last in range(len(sysargs) - 1, -1, -1):
if sysargs[last] == '--':
twillargs = sysargs[last + 1:]
sysargs = sysargs[:last]
sys_args = sys.argv[1:]
if '--' in sys_args:
for last in range(len(sys_args) - 1, -1, -1):
if sys_args[last] == '--':
twill_args = sys_args[last + 1:]
sys_args = sys_args[:last]
break
options, args = parser.parse_args(sysargs)
options, args = parser.parse_args(sys_args)

@@ -356,0 +372,0 @@ if options.show_version:

+1
-0
twill/unit.py

@@ -58,2 +58,3 @@ """Support functionality for using twill in unit tests."""

""""Get the test server URL."""
# noinspection HttpUrlsUsage
return f"http://{HOST}:{self.port}/"

@@ -60,0 +61,0 @@ 

+98
-75
twill/utils.py

@@ -11,8 +11,13 @@ """

from collections import namedtuple
from typing import Any, List, NamedTuple, Optional, Union, Sequence, Tuple
from lxml import html
from requests import Response
from requests.structures import CaseInsensitiveDict
from lxml.html import (
fromstring as html_to_tree, tostring as tree_to_html,
CheckboxGroup, FormElement, HtmlElement, InputElement,
MultipleSelectOptions, RadioGroup, SelectElement, TextareaElement)
try:
import tidylib
import tidylib # type: ignore
except (ImportError, OSError):

@@ -26,6 +31,26 @@ # ImportError can be raised when PyTidyLib package is not installed

__all__ = [
'gather_filenames', 'get_equiv_refresh_interval', 'html_to_tree',
'is_hidden_filename', 'is_twill_filename', 'print_form',
'make_boolean', 'make_int', 'make_twill_filename',
'run_tidy', 'tree_to_html', 'trunc', 'unique_match',
'CheckboxGroup', 'FieldElement', 'FormElement',
'HtmlElement', 'InputElement', 'Link', 'RadioGroup',
'ResultWrapper', 'SelectElement', 'Singleton', 'TextareaElement',
'UrlWithRealm', 'Response']
Link = namedtuple('Link', 'text, url')
FieldElement = Union[
CheckboxGroup, InputElement, RadioGroup, SelectElement, TextareaElement]
class Link(NamedTuple):
text: str
url: str
# Depending on the configuration, realms can be ignored
UrlWithRealm = Union[str, Tuple[str, str]]
class Singleton:

@@ -51,10 +76,10 @@ """A mixin class to create singleton objects."""

"""
def __init__(self, response):
def __init__(self, response: Response) -> None:
self.response = response
self.encoding = response.encoding
try:
self.tree = html.fromstring(self.text)
self.tree = html_to_tree(self.text)
except ValueError:
# may happen when there is an XML encoding declaration
self.tree = html.fromstring(self.content)
self.tree = html_to_tree(self.content)
self.xpath = self.tree.xpath

@@ -64,3 +89,3 @@ self._fix_forms()

@property
def url(self):
def url(self) -> str:
""""Get the url of the result page."""

@@ -70,3 +95,3 @@ return self.response.url

@property
def http_code(self):
def http_code(self) -> int:
"""Get the http status code of the result page."""

@@ -76,3 +101,3 @@ return self.response.status_code

@property
def text(self):
def text(self) -> str:
"""Get the text of the result page."""

@@ -82,3 +107,3 @@ return self.response.text

@property
def content(self):
def content(self) -> bytes:
"""Get the binary content of the result page."""

@@ -88,3 +113,3 @@ return self.response.content

@property
def headers(self):
def headers(self) -> CaseInsensitiveDict:
"""Get the headers of the result page."""

@@ -94,3 +119,3 @@ return self.response.headers

@property
def title(self):
def title(self) -> Optional[str]:
"""Get the title of the result page."""

@@ -103,3 +128,3 @@ try:

@property
def links(self):
def links(self) -> List[Link]:
"""Get all links in the result page."""

@@ -109,3 +134,3 @@ return [Link(a.text_content(), a.get('href'))

def find_link(self, pattern):
def find_link(self, pattern: str) -> Optional[Link]:
"""Find a link with a given pattern on the result page."""

@@ -118,7 +143,10 @@ regex = re.compile(pattern)

def form(self, formname=1):
"""Get the form with the given name on the result page"""
def form(self, name_or_num: Union[str, int] = 1) -> Optional[FormElement]:
"""Get the form with the given name or number on the result page.
Returns None if no such form can be found on the result page.
"""
forms = self.forms
if isinstance(formname, str):
if isinstance(name_or_num, str):

@@ -128,7 +156,7 @@ # first, try ID

form_id = form.get('id')
if form_id and form_id == formname:
if form_id and form_id == name_or_num:
return form
# next, try regex with name
regex = re.compile(formname)
regex = re.compile(name_or_num)
for form in forms:

@@ -141,4 +169,4 @@ name = form.get('name')

try:
formnum = int(formname) - 1
if not 0 <= formnum < len(forms):
num = int(name_or_num) - 1
if not 0 <= num < len(forms):
raise IndexError

@@ -148,5 +176,5 @@ except (ValueError, IndexError):

else:
return forms[formnum]
return forms[num]
def _fix_forms(self):
def _fix_forms(self) -> None:
"""Fix forms on the page for use with twill."""

@@ -156,8 +184,5 @@ # put all stray fields into a form

if orphans:
form = [b'<form>']
for orphan in orphans:
form.append(html.tostring(orphan))
form.append(b'</form>')
form = b''.join(form)
self.forms = html.fromstring(form).forms
form_parts = [b'<form>'] + [
tree_to_html(orphan) for orphan in orphans] + [b'</form>']
self.forms = html_to_tree(b''.join(form_parts)).forms
self.forms.extend(self.tree.forms)

@@ -173,3 +198,3 @@ else:

def trunc(s, length):
def trunc(s: Optional[str], length: int) -> str:
"""Truncate a string to a given length.

@@ -185,3 +210,3 @@

def print_form(form, n):
def print_form(form: FormElement, n: int) -> None:
"""Pretty-print the given form, with the assigned number."""

@@ -198,6 +223,7 @@ info = log.info

value = field.value
if hasattr(field, 'value_options'):
value_options = getattr(field, 'value_options', None)
if value_options:
items = ', '.join(
f"'{getattr(opt, 'name', opt)}'"
for opt in field.value_options)
for opt in value_options)
value_displayed = f'{value} of {items}'

@@ -219,3 +245,3 @@ else:

def make_boolean(value):
def make_boolean(value: Any) -> bool:
"""Convert the input value into a boolean."""

@@ -247,3 +273,3 @@ value = str(value).lower().strip()

def make_int(value):
def make_int(value: Any) -> int:
"""Convert the input value into an int."""

@@ -260,3 +286,3 @@ try:

def set_form_control_value(control, value):
def set_form_control_value(control: FieldElement, value: str) -> None:
"""Set the given control to the given value

@@ -266,19 +292,19 @@

"""
if isinstance(control, html.InputElement):
if isinstance(control, InputElement):
if control.checkable:
try:
value = make_boolean(value)
boolean_value = make_boolean(value)
except TwillException:
# if there's more than one checkbox,
# it should be a html.CheckboxGroup, see below.
# it should be a CheckboxGroup, see below.
pass
else:
control.checked = value
control.checked = boolean_value
elif control.type not in ('submit', 'image'):
control.value = value
elif isinstance(control, (html.TextareaElement, html.RadioGroup)):
elif isinstance(control, (TextareaElement, RadioGroup)):
control.value = value
elif isinstance(control, html.CheckboxGroup):
elif isinstance(control, CheckboxGroup):
if value.startswith('-'):

@@ -295,3 +321,3 @@ value = value[1:]

elif isinstance(control, html.SelectElement):
elif isinstance(control, SelectElement):
# for ListControls we need to find the right *value*,

@@ -308,9 +334,9 @@ # and figure out if we want to *select* or *deselect*

# now, select the value.
options = [opt.strip() for opt in control.value_options]
option_names = [(c.text or '').strip() for c in control.getchildren()]
full_options = dict(zip(option_names, options))
for name, opt in full_options.items():
option_values = [val.strip() for val in control.value_options]
options = control.getchildren() # type: ignore
option_names = [(c.text or '').strip() for c in options]
for name, opt in zip(option_names, option_values):
if value not in (name, opt):
continue
if isinstance(control.value, html.MultipleSelectOptions):
if isinstance(control.value, MultipleSelectOptions):
if add:

@@ -321,6 +347,3 @@ control.value.add(opt)

else:
if add:
control.value = opt
else:
control.value = None
control.value = opt if add else ""
break

@@ -334,3 +357,3 @@ else:

def _all_the_same_submit(matches):
def _all_the_same_submit(matches: Sequence[FieldElement]) -> bool:
"""Check if a list of controls all belong to the same control.

@@ -342,3 +365,3 @@

for match in matches:
if not isinstance(match, html.InputElement):
if not isinstance(match, InputElement):
return False

@@ -355,3 +378,3 @@ if match.type not in ('submit', 'hidden'):

def _all_the_same_checkbox(matches):
def _all_the_same_checkbox(matches: Sequence[FieldElement]) -> bool:
"""Check if a list of controls all belong to the same checkbox.

@@ -366,3 +389,3 @@

for match in matches:
if not isinstance(match, html.InputElement):
if not isinstance(match, InputElement):
return False

@@ -379,3 +402,3 @@ if match.type not in ('checkbox', 'hidden'):

def unique_match(matches):
def unique_match(matches: Sequence[FieldElement]) -> bool:
"""Check whether a match is unique"""

@@ -386,3 +409,3 @@ return (len(matches) == 1 or

def run_tidy(html):
def run_tidy(html: str) -> Tuple[Optional[str], Optional[str]]:
"""Run HTML Tidy on the given HTML string.

@@ -408,4 +431,4 @@

def _equiv_refresh_interval():
"""Get smallest interval for which the browser should follow redirects.
def get_equiv_refresh_interval() -> Optional[int]:
"""Get the smallest interval for which the browser should follow redirects.

@@ -418,3 +441,3 @@ Redirection happens if the given interval is smaller than this.

def is_hidden_filename(filename):
def is_hidden_filename(filename: str) -> bool:
"""Check if this is a hidden file (starting with a dot)."""

@@ -425,3 +448,3 @@ return filename not in (

def is_twill_filename(filename):
def is_twill_filename(filename: str) -> bool:
"""Check if the given filename has the twill file extension."""

@@ -431,26 +454,26 @@ return filename.endswith(twill_ext) and not is_hidden_filename(filename)

def make_twill_filename(name):
def make_twill_filename(name: str) -> str:
"""Add the twill extension to the name of a script if necessary."""
if name not in ('.', '..'):
twillname, ext = os.path.splitext(name)
twill_name, ext = os.path.splitext(name)
if not ext:
twillname += twill_ext
if os.path.exists(twillname):
name = twillname
twill_name += twill_ext
if os.path.exists(twill_name):
name = twill_name
return name
def gather_filenames(arglist):
def gather_filenames(args: Sequence[str]) -> List[str]:
"""Collect script files from within directories."""
names = []
for arg in arglist:
names: List[str] = []
for arg in args:
name = make_twill_filename(arg)
if os.path.isdir(name):
for dirpath, dirnames, filenames in os.walk(arg):
dirnames[:] = [
d for d in dirnames if not is_hidden_filename(d)]
for dir_path, dir_names, filenames in os.walk(arg):
dir_names[:] = [
d for d in dir_names if not is_hidden_filename(d)]
for filename in filenames:
if not is_twill_filename(filename):
continue
filename = os.path.join(dirpath, filename)
filename = os.path.join(dir_path, filename)
names.append(filename)

@@ -457,0 +480,0 @@ else: