Created Sun Jun, 09 2019 at 11:35AM

monitoring a log

while true; 
  grep 'Failed to send email' -n5 sdb2excel_*.log;
  echo "sleeping every 3 hours."; 
  date; sleep $((60 * 60 * 3 )); 

Get public IP

dig TXT +short | awk -F'"' '{ print $2}'

recursively save / backup posix file permissions & restore

To back up permissions in the current directory and its subdirectories recursively:

getfacl -R . >permissions.facl

To restore permissions:

setfacl --restore=permissions.facl

mktime datetime in past or future

date +%Y-%m-%d %H:%M:%S -d  "1 day ago"
date +%Y-%m-%d -d  "1 day ago"

rsync download glob

rsync -av --include="uncle_grist_logs*.json.gz" --exclude='*' receiver5:/mnt/twitter/receiver/trunk/ .

rsync just pm and pl files

rsync -av --include="*/" --include="*.pm" --include="*.pl" --exclude="*" selenium:/mnt/twitter/receiver/branches/archive_manager_humancoding_import_v2/ . --dry-run

rsync just csv files

rsync -av opencpu:/tmp/ --include '*.csv' --exclude '*' .

say when done

while true
  if [ 0`ps -ef|grep perl|grep big_mysql_tablename|wc -l|awk  '{print $1}'` -gt 0 ];
  then clear; echo "working..."; 
    date; sleep 60;
    say "Your processes are done"; break;

remove line from file by regex

# verify identified lines
sed -n '/^2768618:/p' all_identified.txt

# delete those lines
sed -n '/^2768618:/d' all_identified.txt

difference between two files

#  cat t1.txt
# cat t2.txt
cat t2.txt
comm -1 -3 <(sort -u t1.txt) <(sort -u t2.txt)
# z
   -1     suppress lines unique to FILE1
   -2     suppress lines unique to FILE2
   -3     suppress lines that appear in both files

for each line in file; do if said

cat non_movie_blacklist.csv|while read title
echo "searing for [$title]\n"; if [ 0`grep -i -c "$title" all_titles.csv` -gt 0 ];then echo $title >> remove_me_from_all_titles.txt; echo "title: $title found"; fi

strip color control codes from output stdout

sed "s,$(printf '\033')\\[[0-9;]*[a-zA-Z],,g"

loop through array

set -A farm_hosts one two three four;
 # OR # 
farm_hosts=(web03 web04 web05 web06 web07)

for i in ${farm_hosts[@]}; do
        su $login -c "scp $httpd_conf_new ${i}:${httpd_conf_path}"
        su $login -c "ssh $i sudo /usr/local/apache/bin/apachectl graceful"


dumping multiple mysql tables

tables=(customers clients products product_skus product_placements users roles)

for tablename in ${tables[@]}; do
   echo "Dumping $tablename to lp1:/100gb_magnetic/$tablename.sql.7z";
   mysqldump -u wmxdev -h -p2eBevuPU wr_all $tablename | 7z a -si $tablename.sql.7z
set -A test_dates '2017-10-12' '2017-10-09'
set -A test_countries IE RU BR UK US DE

for country in ${test_countries[@]}; do
    for rdate in ${test_dates[@]}; do
        perl bin/adhoc_scripts/ --today=$rdate --country=$country > /tmp/test_orig_${rdate}_${country}.html;
        echo "diff /tmp/test_mod_${rdate}_${country}.html /tmp/test_orig_${rdate}_${country}.html"|tee -a /tmp/test_commands.txt

sed grab range of lines between patterns

sed -n '/^2018\/06\/29/,/^2018\/06\/30/p' find_facebook_posts.log

auto queue

I needed something to identify queues in redis where we may drop tweets for import.

while true; do redis-cli --bigkeys --raw|grep "Biggest list"|cut -d"'" -f2|while read title; do if [ `grep -c $title /tmp/all_auto_queues.csv` -gt 0 ];then echo "Start reading from queue: $title";  ulimit -n 2068; perl bin/ --type=redis2s3_twitter_by_title --server=localhost:6379 --queue_name="$title" --exit_when_done; fi; done; sleep 60; done

when process found do something

while [ 0`ps -ef|grep inventory|grep -c "The Free State of Jones"` -lt 1 ];do echo "not The Free State of Jones.. "; sleep 30; done; echo "The Free State of Jones FOUND"; redis-cli del redis2s3_inventory_backfill;

using iTerm2 - download file to machine

it2dl /mnt/tally/trunk/article_search_add_and_index.log

Git sshfs mounted volume oh-my-zsh issue

The settings "oh-my-zsh.hide-dirty" can be set in gitconfig to disable this.


git config --global oh-my-zsh.hide-status 1 will hide it globally.


git config oh-my-zsh.hide-status 1 will hide it for the current repo.

verify ssh key

ssh-keygen -y -f ~/.ssh/id_rsa

keyboard shortcuts

Shortcut What it does
Ctrl + K kill ->
Ctrl + U kill <-
Ctrl + W kill <- 1 word
Ctrl + Y paste yanked line
Ctrl + R search history
Ctrl + X + E middle of writing a long command.. this pops you into $EDITOR
Alt + . pastes previous commands $arg[1]
## tailing log tail -f replacement

less +F /path/to/logfile

Shift + F (tail -f) Ctrl + C (get out of follow mode)

# networking

### adding static route

sudo route add -net gw netmask

### deleting default route

sudo route del default gw eth0

## encrypting / decrypting files

A simple way to encrypt a single file is with openssl:

use openssl aes-128-cbc

openssl des3 < youfile.txt > yourfile.txt.des3
 # This will prompt you for a passphrase, which you will need to enter later when decrypting the file.

openssl des3 -d < yourfile.txt.des3 > yourfile.txt.decrypted


utility copies files / dirs and sets permissions / modes on them

flag description
-m mode
-T --no-target-directory
-t --target-directory=DIRECTORY copy all SOURCE arguments into DIRECTORY
-C --compare compare each pair of source and destination files, and # in some cases, do not modify the destination at all
sudo install -o root -g root -m 644 -C -T $TSDIR/templates/$devtype/syslog-ng/global.conf /etc/syslog-ng/conf.d/01-threatstop-$devtype-globals.conf

at atq one liner

$ echo "curl -X POST 'http://localhost:5566/export/778'" | at now + 1 min  


test logging a message to syslog local0 facility warn level

sudo logger -p local0.warn 'this is a test rpz'

rsyslog to log rpz messages . cat /etc/rsyslog.d/99-named-rpz.conf

if $programname == 'named' and $syslogfacility-text == 'local0' and $msg contains 'rpz' then /var/log/rpz.log

if logger fails with permission denied (ubuntu) I found this worked..

sudo rm /dev/log; sudo ln -s /run/systemd/journal/dev-log /dev/log

delete file by inode number

ls -il {file-name}

 # Use find command to remove file:

find . -inum [inode-number] -exec rm -i {} \;

reset positional args / params

echoMsg WARN "before ARGS: $*";
for opt in $@; do
    if [ `echo "$opt"|grep -c '\-\-'` -gt 0 ]; then

        echoMsg DEBUG "Removing long-style opt ($opt) from command-line args"
        # unfortunately isn't passed spaced tokens i.e. "file name.txt"
        # if [ `echo "$opt"|grep -c ' '` -gt 0 ]; then
        #     new_args="${new_args} \"$opt\""
        # fi
        echoMsg DEBUG "passing $opt along";
        new_args="${new_args} $opt"
echoMsg WARN "ARGS after shifts: $*";
set -- ${new_args}
echoMsg WARN "ARGS after set: $*";

sudo no password

To give a user or group sudo permissions with no password verification.

 #group wheel

Get unique lines in a file

I had an experience on a customers box where it was out of disk space w/a 40GB log file. Doing a cat file | sort -u wasn't an option, however I found the following simple one liner that did the trick.

awk '!seen[$0]++' file


killing a screen session.

$ screen -X -S [session # you want to kill] quit

using screen to connect to a serial device (networking equipment).

sudo screen /dev/ttyUSB0
# if you have a cisco usb direct console device
sudo screen /dev/ttyACM0 9600

Name screen sessions

# name the screen session something meaningful
screen -S "log_server"
# then later after you've detached the screen session you can connect back up via:
screen -r "log_server"