АРХИВ ПО ТЭГУ "СКРИПТЫ"
НОЯБРЬ 14, 2014

Screen Shot 2014 11 13 at 23.00.36 Lottery numbers generator script (bash) %d1%82%d0%b5%d1%85%d0%bd%d0%be%d0%bb%d0%be%d0%b3%d0%b8%d0%b8

Привет,

Решил вот сгенерировать скриптец на баше и поучаствовать в национальной лотерее Великобритании.

#!/bin/bash
  1.  
  2. # Print usage info
  3. function usage() {
  4.   echo "Usage: $0 "
  5.   echo " * lotto   — Lotto"
  6.   echo " * eurom   — Euro Millions"
  7.   echo " * tball   — Thunderball"
  8.   echo " * hotpick — Lotto Hotpicks"
  9.   echo "More info about UK National Lottery visit https://www.national-lottery.co.uk/games/lotto"
  10. }
  11.  
  12. # Check input params
  13. if [ -z "$1" ]; then
  14.   usage
  15.   exit 1
  16. fi
  17.  
  18. type=$1
  19.  
  20. # Generate random number from 1 up to {limit} and print it
  21. function gen_num() {
  22.   limit=$1
  23.   echo -n $[1 + $[ RANDOM % $limit ]]" "
  24. }
  25.  
  26. # Main app
  27. echo -n "Your lucky numbers for $type are: "
  28. case $type in
  29. lotto)
  30.   for i in {1..6}; do gen_num 59; done
  31.   ;;
  32. eurom)
  33.   for i in {1..5}; do gen_num 50; done
  34.   echo -n "+ lucky stars "
  35.   for i in {1..2}; do gen_num 11; done
  36.   ;;
  37. tball)
  38.   for i in {1..5}; do gen_num 39; done
  39.   echo -n "+ thunderball "
  40.   for i in {1..1}; do gen_num 14; done
  41.   ;;
  42. hotpick)
  43.   for i in {1..5}; do gen_num 49; done
  44.   ;;
  45. *)
  46.   usage
  47.   ;;
  48. esac
  49.  
  50. echo ""
  51. echo "Good luck icon smile Lottery numbers generator script (bash) %d1%82%d0%b5%d1%85%d0%bd%d0%be%d0%bb%d0%be%d0%b3%d0%b8%d0%b8 "
СЕНТЯБРЬ 11, 2014

Hi,

Just a note for myself for fast creating of AWS EMR cluster with Custom JAR job.

  1. #!/bin/bash
  2.  
  3. jobName="MyReportJob"
  4. clusterName="EmrCluster1"
  5. clusterId=""
  6. activeStepStates="—step-states PENDING RUNNING"
  7.  
  8. bucket="s3://emr-test"
  9. region="us-west-2"
  10. instanceType="m1.medium"
  11. instanceNumber=3
  12.  
  13. resdir="resources/"
  14. filesPattern='*.log'
  15.  
  16. jarPath="my-job-1.0-SNAPSHOT-jar-with-dependencies.jar"
  17. jarRename="my-job-1.0-SNAPSHOT-wdeps.jar"
  18. jarRenamePath="/tmp/$jarRename"
  19.  
  20. # Exit script gently
  21. trap "on_trap" SIGHUP SIGINT SIGTERM
  22. function on_trap() {
  23.     echo "Exiting script as of signal…"
  24.     exit 1
  25. }
  26.  
  27. # Run command securely
  28. function do2() {
  29.     echo "Running command: [$*]"
  30.     $*
  31.     res=$?
  32.     if [ $res -gt 0 ]; then
  33.         echo "ERROR: Command execution failed."
  34.         exit $res
  35.     fi
  36. }
  37.  
  38. # Prepare data
  39. function prepare_data() {
  40.     # Clear previous run results
  41.     for i in logs results; do
  42.         do2 aws s3 rm "$bucket/$i/" —recursive
  43.     done
  44.  
  45.     # Upload log files
  46.     for i in $(ls "$resdir"/$filesPattern); do
  47.         do2 aws s3 cp "$i" "$bucket/data/"$(basename "$i" "$resdir")
  48.     done
  49.  
  50.     # Upload Hadoop job
  51.     do2 rm -f "$jarRenamePath"
  52.     do2 cp "$jarPath" "$jarRenamePath"
  53.     do2 aws s3 cp "$jarRenamePath" "$bucket/job/$jarRename"
  54. }
  55.  
  56. # Create EMR Hadoop cluster
  57. function create_cluster() {
  58.     clusterId=$(aws emr list-clusters —active | grep "$clusterName" | awk '{print $2}')
  59.     if [ -z "$clusterId" ]; then
  60.         echo "Creating $clusterName cluster. Doesn't exists yet."
  61.         # Start Hadoop cluster
  62.         do2 aws emr \
  63.         create-cluster \
  64.         —no-auto-terminate \
  65.         —region "$region" \
  66.         —instance-count $instanceNumber \
  67.         —instance-type "$instanceType" \
  68.         —name "$clusterName" \
  69.         —ami-version 3.0.4 \
  70.         —hadoop-version 2.2.0 \
  71.         —log-uri "$bucket/logs"
  72.  
  73.         clusterId=$(aws emr list-clusters —active | grep "$clusterName" | awk '{print $2}')
  74.         if [ -z "$clusterId" ]; then
  75.             echo "ERROR Cannot create $clusterName cluster: Cannot get cluster ID. Try 'aws emr —list-clusters'."
  76.             exit 1
  77.         fi
  78.     else
  79.         echo "Do not create $clusterName cluster as it's already exists."
  80.     fi
  81. }
  82.  
  83. # Run Hadoop job
  84. function run_app() {
  85.     echo "Running job $jobName for $clusterName [$clusterId] cluster…"
  86.  
  87.     if [ -z "$clusterId" ]; then
  88.         echo "ERROR No cluster ID found [$clusterId]. Try 'aws emr list-clusters'."
  89.         exit 1
  90.     fi
  91.  
  92.     # Check for JAR existance
  93.     jarStepId=$(aws emr list-steps —cluster-id "$clusterId" $activeStepStates | grep "$jobName" | awk '{print $3}')
  94.     if [ ! -z $jarStepId ]; then
  95.         echo "WARN Not going to add Custom JAR step $jobName because it's already exists and is in active state. Try 'aws emr list-steps —cluster-id $clusterId $activeStepStates'."
  96.         return
  97.     fi
  98.  
  99.     # Run uploaded job in Hadoop cluster
  100.     do2 aws emr \
  101.     add-steps \
  102.     —cluster-id "$clusterId" \
  103.     —steps \
  104.     Type=CUSTOM_JAR,Name=$jobName,ActionOnFailure=CANCEL_AND_WAIT,Jar=$bucket/job/$jarRename,Args=$bucket/data,$bucket/results
  105.  
  106.     jarStepId=$(aws emr list-steps —cluster-id "$clusterId" $activeStepStates | grep "$jobName" | awk '{print $3}')
  107.     if [ -z $jarStepId ]; then
  108.         echo "ERROR Cannot find/create Custom JAR step $jobName. Try 'aws emr list-steps —cluster-id $clusterId $activeStepStates'."
  109.         exit 2
  110.     fi
  111.  
  112.     echo "Custom JAR $jobName with step ID [$jarStepId] on cluster $clusterName [$clusterId] has been successfully created."
  113. }
  114.  
  115. # Business logic
  116. if [ $1 = "-d" ]; then
  117.     echo "Preparing data for the job(s)…"
  118.     prepare_data
  119. else
  120.     echo "Do not prepare data for the job(s)."
  121. fi
  122.  
  123. create_cluster
  124. run_app
ИЮЛЬ 31, 2014

Support multiple (two) directories.

Original file modification date is kept.

Old files are kept as well.

  1.  
  2. #!/bin/bash
  3.  
  4. dir1=a
  5. dir2=bbb
  6. resdir=results
  7.  
  8. # Save file name separator
  9. OLDIFS=$IFS
  10. IFS=$'\n'
  11.  
  12. # File rename by date
  13. echo "Start first processing…"
  14. for fl in $(find "$dir1" "$dir2" -type f); do
  15.     echo " * processing 1: $fl"
  16.     dirpostfix=a
  17.     if [ $(dirname "$fl") == "$dir2" ]; then
  18.         dirpostfix=b
  19.     fi
  20.  
  21.     newfileBase=$(basename "$fl")
  22.     newfile=$(stat -f '%Sm' -t "%Y%m%d%H%M.%S_${dirpostfix}_${newfileBase}" "$fl")
  23.     cp "$fl" "$resdir"/"$newfile"
  24. done
  25. ls -alh "$resdir/"
  26.  
  27. # Process created files
  28. echo ""
  29. echo "Start second processing…"
  30. pushd "$resdir"
  31. i=1
  32. for fl in $(ls); do
  33.     echo " * processing 2: $fl"
  34.  
  35.     fts=$(echo $fl | cut -c3-15)
  36.     newfile=$(seq -f "%03g" $i $i)"_"${fl:16}
  37.    
  38.     mv "$fl" "$newfile"
  39.     touch -t "$fts" "$newfile"
  40.  
  41.     i=$(expr $i + 1)
  42. done
  43. popd
  44.  
  45. # Restore file name separator
  46. IFS=$OLDIFS
  47.  
  48. ls -alh "$resdir"
  49.  
  50. echo ""
  51. echo "Done! Enjoy icon smile File rename with numeric postfix bash script %d1%82%d0%b5%d1%85%d0%bd%d0%be%d0%bb%d0%be%d0%b3%d0%b8%d0%b8 "
  52. echo ""

ИЮЛЬ 16, 2011

Хаи.

Обожаю гугл.

30 минутный php скрипт, который грабит исторические данные с Google Financial по всем стакам в период с 1 января 1995 по 1 июля 2011 и сохраняет в CSV файлики вида <CUR_DIR>/<MarketName>/<Ticker>.csv

Внутри данные OCHL + Volume.

Я его правда еще не запускал на CID до 30к, но думаю там должно в итоге получиться не мало стаков.

date_default_timezone_set('UTC');
  1.  
  2. define('GRABBER_PATH', dirname(__FILE__));
  3.  
  4. // Jan-1-1995 up to Jul-1-2011
  5. $url = 'http://www.google.com/finance/historical?cid=!CID!&startdate=Jan+1%2C+1995&enddate=Jul+1%2C+2011&num=30';
  6. $exportUrlPostfix = '&output=csv';
  7.  
  8. // The easiest way
  9. $pattern = '/\<title\>;([^:]+): ([\w+]+):([\w\d]+) historical price/';
  10.  
  11. // CID up to 30k 'cause not every cid is an existings stock icon sad Исторические данные акций NYSE и NASDAQ (дневные) %d1%82%d1%80%d0%b5%d0%b9%d0%b4%d0%b8%d0%bd%d0%b3 %d1%82%d0%b5%d1%85%d0%bd%d0%be%d0%bb%d0%be%d0%b3%d0%b8%d0%b8
  12. $cidStart = 1;
  13. $cidEnd = 30000;
  14.  
  15. for ($cid = $cidStart; $cid < $cidEnd; $cid++) {
  16.         $curUrl = str_replace('!CID!', $cid, $url);
  17.  
  18.         $pageHtml = @file_get_contents($curUrl, false, null, 0, 1000);
  19.         if (empty($pageHtml)) {
  20.                 continue;
  21.         }
  22.  
  23.         if (!preg_match($pattern, $pageHtml, $matches)) {
  24.                 continue;
  25.         }
  26.  
  27.         $companyName = $matches[1];
  28.         $market = $matches[2];
  29.         $ticker = $matches[3];
  30.  
  31.         echo "[CID=$cid] Found: $companyName ($market:$ticker)\n";
  32.  
  33.         if (!file_exists(GRABBER_PATH . '/' . $market)) {
  34.                 mkdir(GRABBER_PATH . '/' . $market);
  35.         }
  36.  
  37.         if (!is_dir(GRABBER_PATH . '/' . $market)) {
  38.                 echo " * ERR: " . GRABBER_PATH . '/' . $market . " is not dir\n";
  39.                 continue;
  40.         }
  41.  
  42.         $downloadUrl = $curUrl . $exportUrlPostfix;
  43.         $csvData = @file_get_contents($downloadUrl);
  44.         if (empty($csvData)) {
  45.                 echo " * ERR: Cannot grab data from $downloadUrl\n";
  46.                 continue;
  47.         }
  48.  
  49.         $csvFile = GRABBER_PATH . '/' . $market . '/' . $ticker . '.csv';
  50.         if (!file_put_contents($csvFile, $csvData)) {
  51.                 echo " * ERR: Cannot save CSV data to $csvFile\n";
  52.                 continue;
  53.         }
  54. }