Export Python automation results to DataDog

  • The following Python script helps you to update the automation results right after the execution on exit
from datadog import initialize, api
import time, os, glob, json

print os.environ['API_KEY']

options = {
  'api_key': os.environ['API_KEY']
}

initialize(**options)

stats = json.load('./stats.json')

api.Metric.send(metric="qa.baseline.desktop.passed", points=stats['passed'], tags=["country:us")
api.Metric.send(metric="qa.baseline.desktop.failed", points=stats['failed'], tags=["country:us")
api.Metric.send(metric="qa.baseline.desktop.skipped", points=stats['skipped'], tags=["country:us")
  • Import the following Dashboard json with the basic settings required


{
"title":"QA Dashboard",
"description":"",
"widgets":[
{
"id":0,
"definition":{
"type":"timeseries",
"requests":[
{
"q":"avg:qa.baseline.desktop.passed{country:ae}",
"display_type":"bars",
"style":{
"palette":"cool",
"line_type":"solid",
"line_width":"normal"
}
},
{
"q":"avg:qa.baseline.desktop.failed{country:ae}",
"display_type":"bars",
"style":{
"palette":"orange",
"line_type":"solid",
"line_width":"normal"
}
},
{
"q":"avg:qa.baseline.desktop.passed{country:us}",
"display_type":"bars",
"style":{
"palette":"cool",
"line_type":"solid",
"line_width":"normal"
}
},
{
"q":"avg:qa.baseline.desktop.failed{country:us}",
"display_type":"bars",
"style":{
"palette":"orange",
"line_type":"solid",
"line_width":"normal"
}
}
],
"yaxis":{
"label":"",
"scale":"linear",
"min":"auto",
"max":"auto",
"include_zero":true
},
"title":"DESKTOP BASELINE",
"title_size":"16",
"title_align":"center",
"show_legend":false
},
"layout":{
"x":0,
"y":0,
"width":60,
"height":22
}
},
{
"id":1,
"definition":{
"type":"query_value",
"requests":[
{
"q":"avg:qa.baseline.desktop.failed{*}",
"aggregator":"sum",
"conditional_formats":[
{
"comparator":">",
"value":0,
"palette":"white_on_red"
},
{
"comparator":"<",
"value":1,
"palette":"white_on_green"
}
]
}
],
"title":"DESKTOP",
"title_size":"16",
"title_align":"center",
"autoscale":false,
"precision":0,
"text_align":"center"
},
"layout":{
"x":0,
"y":23,
"width":15,
"height":22
}
},
{
"id":2,
"definition":{
"type":"query_value",
"requests":[
{
"q":"avg:qa.baseline.mdot.failed{*}",
"aggregator":"last",
"conditional_formats":[
{
"comparator":">",
"value":0,
"palette":"white_on_red"
},
{
"comparator":"<",
"value":1,
"palette":"white_on_green"
}
]
}
],
"title":"MDOT",
"title_size":"16",
"title_align":"center",
"autoscale":false,
"precision":0,
"text_align":"center"
},
"layout":{
"x":61,
"y":23,
"width":15,
"height":22
}
},
{
"id":3,
"definition":{
"type":"timeseries",
"requests":[
{
"q":"avg:qa.baseline.mdot.passed{country:ae}",
"display_type":"bars",
"style":{
"palette":"cool",
"line_type":"solid",
"line_width":"normal"
}
},
{
"q":"avg:qa.baseline.mdot.failed{country:ae}",
"display_type":"bars",
"style":{
"palette":"orange",
"line_type":"solid",
"line_width":"normal"
}
},
{
"q":"avg:qa.baseline.mdot.passed{country:us}",
"display_type":"bars",
"style":{
"palette":"cool",
"line_type":"solid",
"line_width":"normal"
}
},
{
"q":"avg:qa.baseline.mdot.failed{country:us}",
"display_type":"bars",
"style":{
"palette":"orange",
"line_type":"solid",
"line_width":"normal"
}
}
],
"yaxis":{
"label":"",
"scale":"linear",
"min":"auto",
"max":"auto",
"include_zero":true
},
"title":"MDOT BASELINE",
"title_size":"16",
"title_align":"center",
"show_legend":false
},
"layout":{
"x":61,
"y":0,
"width":60,
"height":22
}
},
{
"id":4,
"definition":{
"type":"toplist",
"requests":[
{
"q":"max:qa.baseline.desktop.passed{*} by {country}"
}
],
"title":"Desktop Test Counts",
"title_size":"16",
"title_align":"center"
},
"layout":{
"x":16,
"y":23,
"width":44,
"height":22
}
},
{
"id":5,
"definition":{
"type":"toplist",
"requests":[
{
"q":"max:qa.baseline.mdot.passed{*} by {country}"
}
],
"title":"MDot Test Counts",
"title_size":"16",
"title_align":"center"
},
"layout":{
"x":77,
"y":23,
"width":44,
"height":22
}
}
],
"template_variables":[
{
"name":"",
"default":"*",
"prefix":null
}
],
"layout_type":"free",
"is_read_only":false,
"notify_list":[
],
"id":"hpe-h5g-wut"
}

view raw

dashboard.json

hosted with ❤ by GitHub

  • Thats it!

Get appActivity and appPackage name from an unknown apk file

Method #1

  • Open Android Emulator
  • Copy and paste the apk file inside platform-tools folder
  • If you’ve installed SDK through ‘Android Studio’ in MAC, here is the location where you need to copy the .apk file
/Users/$(whoami)/Library/Android/sdk/platform-tools
  • Open terminal and install your apk file in emulator
adb install app.apk
  • Finally obtain the appActivity and appPackage names
adb shell "dumpsys window windows | grep -E 'mCurrentFocus|mFocusedApp'"

Method #2

  • Copy and paste the .apk file inside build-tools folder
/Users/$(whoami)/Library/Android/sdk/build-tools/29.0.0
  • Open terminal and run
./aapt dump badging app.apk
  • Observe the appPackage name and appActivity name separately

Linux cmds to troubleshoot DNS server

NSLOOKUP

Troubleshoot DNS related issues

# get ip address of the host name
nslookup www.google.com

# get dns server name
nslookup 

# get mail server name
nslookup -querytype=MX www.google.com

$ nslookup
> set type=MX
> www.google.com

# get name server name
nslookup -querytype=NS www.google.com

$ nslookup
> set type=NS
> www.google.com

# switch default local Dserver to another server
$ nslookup
> server 8.8.8.8
> www.google.com

# switch back to default type
nslookup -querytype=A www.google.com

$ nslookup
> set type=A
> www.google.com

DIG

dig is the short form for Domain Information Groper; it is also used for troubleshooting DNS related issues

dig www.prashanthsams.com

# dig in short
dig www.prashanthsams.com +short
# debug in detail
dig +trace www.prashanthsams.com

Useful Linux Commands

WGET

Downloads a single file/folder or an entire site in a single cmd

# download entire html including images
wget -E -H -k -p https://www.google.com

# continue download from the middle
wget -c http:///.zip

# download in background
wget -b http:///.zip

# download entire website and its contents including landing pages
wget --mirror -p --convert-links -P ./folder http://www.google.com

cURL

Download multiple files through URL/links

# -o is for output and google.html is the output file & format
curl -o google.html https://www.google.com

# -# shows the progress
curl -# -o google.com https://www.google.com

# -k ignores the ssl certificate shows the progress
curl -k -o google.com https://www.google.com

CHMOD

Individual 7 (rwx/4+2+1) is a combination of permissions 4, 2, and 1; where, 4 – read, 2 – write, and 1 – execute

# check if the file(s) have read/write access
ls -la
ls -l 

# give read, write, and execute access to user, group, and others
chmod 777 
chmod a=rwx 

# give read, write, and execute access to all the files under a folder recursively
chmod -R 777 

# check if the files have read/write access
# USER
chmod u=rw 
chmod 500 
# GROUP
chmod g=rw 
chmod 050 
# OTHER
chmod o=rw 
chmod 005 

UID

UID helps you switch between users

[LINUX]
# list users with uid
cat /etc/passwd

[MAC]
# lists users 
dscl . -list /Users

# list users with uid
dscl . -list /Users UniqueID 

# get specific user id
dscl . -read /Users/prashanthrajjapa UniqueID

GREP

Global regular expression parser/print

# list all the matching files with text
ls * | grep 

# print the lines containing given text
grep --color  

# print the lines containing given text with line number
grep --color -n  

# print the lines containing with text from all the matching files
grep --color  *

# ignore case-sensitive
grep --color -i  

# search in all the sub directories recursively
grep --color -i -r  

# match any character in the middle using .*
grep --color  *

WC

To know the number of word counts, lines, and characters in a file

# array 1 represents total number of lines
# array 2 represents total number of words
# array 3 represents total number of characters
wc 
wc < 

# LINES
wc  -l
# WORDS
wc  -w
# CHARS
wc  -c

PS

To view the status of the running processes

# lists all the running process with PID number
ps -A
ps -A | less

# BSD syntax to list all process with full details (owner, %MEM, start time, PID, etc.,)
ps axu
ps aux
ps axu | grep firefox

# check specific process run
ps -A | grep firefox

# check process start time
ps -ef | grep firefox
ps -ef | grep firefox | less

KILL

To kill the running process

# kill a process using process ID
kill 

# kill a process using name
pkill firefox

# force kill a process using name
killall -9 
(e.g., killall -9 chromedriver)

SYMLINK

Set pseudo path for the existing one

# create symlink
ln -s  
(e.g., ln -s /Applications/Firefox.app/ firefox)

# create/update symlink
ln -sf  

TTY

Print current terminal source

# create symlink
tty

TOP

Troubleshoot CPU memory usage

# check instant cpu memory usage live
top -F -R -o cpu

# check instant cpu memory usage live
top -o rsize

HTOP

Advanced version of top; it has to installed externally and not comes by default

# check instant cpu memory usage live
sudo apt-get install htop
htop

AWK

Helps you write logic in the cmd line and play with text

# print whole line 
ls -la | awk '{print $0}'

# print first field
ls -la | awk '{print $1}'
ls -l . | grep "^-" | awk '{print $1}'

UNIQ

Simplify your work by getting line count, duplicates and uniqueness from a file

# prints all the unique lines in a file
uniq -u your_text_file.format

# prints all the duplicate lines in a file
uniq -u your_text_file.format

# prints all the lines with duplicate counts in a file 
uniq -c your_text_file.format

COMPRESS

Zip and unzip files

# compress a file 
compress file_name.format

# compress a file and print percentage reduction in size
compress -v file_name.format

# uncompress a file
uncompress file_name.format.Z

# uncompress a file and print percentage reduction in size
uncompress -v file_name.format.Z

PRINT FILE SIZE 

Print file size

# readable format to see the size of file
ls -lh file_name.format

# alternate way to see the file size
du -sh file_name.format

FIND

Locate directory(ies) and it’s files in the local machine

find ~/ directory_name 

DB Connect

Command line DB conenction

# mysql db connect
mysql -u username -ppassword -h hostname
mysql> show databases;
mysql> use dbname;
mysql> show tables;

# mysql db connect with port and dbname
mysql -u username -ppassword -h hostname -P port -D dbname

PS1 – Primary Prompt

Run set of cmds on launching terminal/cmd prompt. PS1 is the primary cmd prompt, PS2 is the secondary cmd prompt, and PS3 is the least used cmd prompt.

# ps1 to change the terminal color 
LIGHT GREEN
export PS1="\e[0;32m\e[6m\u@\h \w> \e[m "
BRIGHT GREEN
export PS1="\e[0;32m\e[1m\u@\h \w> \e[m "
UNDERLINE GREEN
export PS1="\e[0;36m\e[4m\u@\h \w> \e[m "
BACKGROUND GREEN
export PS1="\e[7;32m\e[1m\u@\h \w> \e[m "
BLINK GREEN
export PS1="\e[5;32m\e[1m\u@\h \w> \e[m "
VALUE IN GREEN
export PS1="\e[0;32m\e[1m\u@\h \w> \e[5m "
STANDARD GREEN
export PS1="\e[1;32m\e[1m\u@\h \w> \e[0m "
more details..
  • To make the above cmds effective, try copy & paste inside bash_profile
# set as environment variable
sudo vi ~/.bash_profile
export PS1="\e[1;32m\e[1m\u@\h \w> \e[0m "

Ruby Database cleaner (MySQL)

Database cleaner helps you clean database for tests

  • Install and config database_cleaner

Gemfile

gem 'database_cleaner'
gem 'sequel'

env.rb

require 'database_cleaner'
require 'sequel'
  • DB connection through database_cleaner
connection = Sequel.connect("mysql2://username:password@hostname:port/db_name")
DatabaseCleaner[:sequel].db = connection
  • Alternate option for DB connect
DatabaseCleaner[:sequel, {:connection => Sequel.connect('mysql2://username:password@hostname:port/db_name')}]
  • Check if the connection is ready
DatabaseCleaner.connections.inspect
  • Set transaction as the strategy; however, this works only if the table doesn’t have foreign key constraints
DatabaseCleaner.strategy = :transaction
DatabaseCleaner.start
<dirty the db>
DatabaseCleaner.clean
  • To clean the complete table, clean the db with truncation
DatabaseCleaner.clean_with(:truncation)
  • To ignore foreign key constraints,
connection.run('SET FOREIGN_KEY_CHECKS=0;')
DatabaseCleaner.clean_with(:truncation)
connection.run('SET FOREIGN_KEY_CHECKS=1;')

 

 

 

Handle Basic authentication in Selenium for Chrome browser

Chrome extension

Create a simple chrome extension with custom username and password defined in it.

  • Create a .js file, background.js with the custom username and password
var username = "your_user";
var password = "your_pass";
var retry = 3;

chrome.webRequest.onAuthRequired.addListener(
  function handler(details) {
    if (--retry < 0)
      return {cancel: true};
    return {authCredentials: {username: username, password: password}};
  },
  {urls: ["<all_urls>"]},
  ['blocking']
);

[Text in red has to be customized specific to the project]

  • Create a .json file, manifest.json as shown below
{
  "manifest_version": 2,
  "name": "Authentication for ...",
  "version": "1.0.0",
  "permissions": ["<all_urls>", "webRequest", "webRequestBlocking"],
  "background": {
    "scripts": ["background.js"]
  }
}
  • Now, zip both the files and rename it to extension.crx

  • To check if the custom extension works fine, open a new tab in Google-Chrome and enter chrome://extensions/
  • Now, select the folder where both the manifest.json and background.js files were located (on clicking Load unpacked extension…)

  • Once the extension is loaded, you will observe the newly added icon next to the browser’s address bar; now, try loading the web application that asks you for the basic auth

Chrome options

Configure the newly created .crx chrome extension through ChromeOptions as described below

  • Ruby snippet to configure chrome extension through ChromeOptions
options = {
  'chromeOptions' => {
    'extensions' => [
      Base64.strict_encode64(File.open("../extension.crx", 'rb').read)
    ]
  }
}

caps = Selenium::WebDriver::Remote::Capabilities.chrome(options)
@driver = Selenium::WebDriver.for :chrome, desired_capabilities: caps
  • The above snippet works fine in GUI mode; however, through browser’s headless mode, it doesn’t
  • Linux users, try xvfb to overcome this issue; for Ruby, you need a third party client/lib, headless to couple with Linux’s xvfb mode
gem 'headless'
require 'headless'
  • Start headless support before initiating the tests
@headless = Headless.new
@headless.start
  • And terminate the headless mode at exit
@headless.destroy
  • In Jenkins, you need to set the environment ready for xvfb to run tests without GUI mode
sudo yum install xorg-x11-server-Xvfb -y
(or)
sudo apt-get install xorg-x11-server-Xvfb -y
export DISPLAY=:0
  • Meanwhile, go to Manage Jenkins > Configure System; add manual export for DISPLAY :0 in the global properties itself (this step is optional)

  • Force kill Xvfb if previously running
pkill -f Xvfb || true
  • To set headless browser window size, you can still use selenium-webdriver’s native way
target_size = Selenium::WebDriver::Dimension.new(1600, 1268)
@driver.manage.window.size = target_size

 

Encrypt and Decrypt passwords in Ruby

Active Support

To encode and decode a text string, ruby offers a number of libraries. Let’s see how Activesupport works here,

  • Install activesupport gem
gem 'activesupport'
require 'active_support'
  • Snippet for encryption and decryption
key = SecureRandom.random_bytes(32)
crypt = ActiveSupport::MessageEncryptor.new(key)
encrypted_pwd = crypt.encrypt_and_sign("your actual password")
crypt.decrypt_and_verify(encrypted_pwd)

 

Base64

Base64 is used to encode and decode a value. It’s an alternate and easiest way to encrypt and decrypt values.

  • Snippet for Base64 encryption and decryption
def encrypt(input_string)
  Base64.encode64(input_string)
end

def decrypt(encrypted_string)
  Base64.decode64(encrypted_string)
end
k = encrypt('555')
=> "NTU1\n"
decrypt(k)
=> "555"

 

MessageEncryptor

  • Snippet for MessageEncryptor encryption and decryption
len = ActiveSupport::MessageEncryptor.key_len
=> 32
salt = SecureRandom.random_bytes(len)
key = ActiveSupport::KeyGenerator.new('password').generate_key(salt, len)
crypt = ActiveSupport::MessageEncryptor.new(key)
encrypted_data = crypt.encrypt_and_sign('777')
=> T0VjQVRtS0MrNno.....a8b8d980d9f1b35343e 
crypt.decrypt_and_verify(encrypted_data)
=> 777

 

 

Flush Redis cache using Ruby client

Flush all the redis cache keys

  • Install redis ruby client library
gem install redis
require 'redis'
  • Include the method, redis_connect before each test scenario in-order to clear redis cache.
redis_connect("your_host_name", xxxx)
def redis_connect(host, port)
 redis = Redis.new(host: "#{host}", port: port)
 redis.flushall
end
  • The same can be achieved with Linux library in terminal, redis-cli.
  • Install redis-cli
sudo yum install -y gcc
wget http://download.redis.io/redis-stable.tar.gz && tar xvzf redis-stable.tar.gz && cd redis-stable && make
sudo cp src/redis-cli /usr/bin/
redis-cli -h your_host_name flushall

Delete specific redis key from the list 

def redis_connect(host, port)
  redis = Redis.new(host: "#{host}", port: port)

  redis.keys.each do |k|
    
    if k.size == 10
      redis.del k
    end

    if k.include? "text"
      redis.del k
    end

  end
end

Configure Selenium Jenkins job with Ruby Env setup from Jenkins User

Note: (Follow this post before proceeding)

  • Start with creating a new Jenkins job by clicking Jenkins > New Item
  • Enter the job name, select Freestyle project, and click ok

  • Choose Source Code Management > Git and enter the prompt GitHub repo. Enter the branch name if different.

  • Now, select Build > Add build step > Execute shell

  • And follow these steps in shell to run selenium cucumber tests
  1. Set temporary RVM environment setup
  2. source /var/lib/jenkins/.rvm/bin/rvm

    or

    source ~/.rvm/bin/rvm
  3. Create rvm gemset and switch to it
  4. rvm gemset create test
    rvm gemset use test
  5. Install the ruby libraries
  6. gem install bundler
    bundle install
  7. Run Selenium cucumber tests
  8. cucumber features/scenarios/**/*.feature
  9. Finally, apply changes

Skip the manual temporary RVM environment setup [Optional]

  • Go to Manage Jenkins > Manage Plugins, click on the Available tab
  • Enter rvm in the filter

  • Select checkbox, download & restart Jenkins to take effect
  • Now, open the job created earlier and click Configure

  • you’ll see an extra newly added option, Run the build in a RVM-managed environment under Build Environment section

  • Add the installed ruby version in it (say, ruby-2.4.1) and remove source ~/.rvm/bin/rvm from the Execute shell

 

Install RVM and Ruby on Amazon Linux as Jenkins/Root User

Install RVM and Ruby as a Jenkins User

  • Install Jenkins (Follow this link)
  • Become a root user
sudo su
  • Install prerequisites for RVM and Ruby
yum install -y gcc openssl-devel libyaml-devel libffi-devel readline-devel zlib-devel gdbm-devel ncurses-devel ruby-devel gcc-c++ jq git patch autoconf automake bison libtool patch sqlite-devel
  • Set password for Jenkins as a root user (if needed)
sudo passwd jenkins

  • Switch as Jenkins User

  • Import GPG key before RVM install
curl -sSL https://rvm.io/mpapis.asc | gpg2 --import -

[run the cmd if the above didn't work]
gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
  • Install RVM
curl -sSL https://get.rvm.io | bash -s stable --ruby
  • Export temporary setup
source /var/lib/jenkins/.rvm/scripts/rvm
  • Verify RVM and Ruby installations
rvm -v && which rvm
ruby -v && which ruby

Install RVM and Ruby as a Root User [optional]

  • Install Jenkins (Follow this link)
  • Become a root user
sudo su
  • Install prerequisites for RVM and Ruby
yum install -y gcc openssl-devel libyaml-devel libffi-devel readline-devel zlib-devel gdbm-devel ncurses-devel ruby-devel gcc-c++ jq git
  • Import GPG key before RVM install
curl -sSL https://rvm.io/mpapis.asc | gpg2 --import -
  • Install RVM
curl -sSL https://get.rvm.io | bash -s stable --ruby
  • Now, run the temporary RVM environment for effect
source /usr/local/rvm/scripts/rvm
  • Verify RVM and Ruby installations
rvm -v && which rvm
ruby -v && which ruby