GitPrep
Login
Showing 2 changed files with 154 additions and 0 deletions
+4
README.md
View
... ... 
@@ -1,3 +1,7 @@
1 1 
 # pcurl
2 2
3 3 
 Modified version of https://sourceforge.net/projects/pcurl/
4 
+
5 
+- Added Progress information
6 
+- Number of download segment is tweakable within the script itself (MAX_SEGMENTS)
7 
+- By default 20 segments are downloaded in parallel
+150
pcurl
View
... ... 
@@ -0,0 +1,150 @@
1 
+#!/bin/bash
2 
+#title           :pcurl.sh
3 
+#description     :This script will download a file in segments.
4 
+#author		 :Phil Rumble prumble@au1.ibm.com
5 
+#date            :20120202
6 
+#version         :0.1
7 
+#usage		 :bash pcurl.sh url
8 
+#notes           :Install curl to use this script.
9 
+
10 
+MAX_SEGMENTS=20
11 
+
12 
+debug() { echo "DEBUG: $*" >&2; }
13 
+
14 
+
15 
+waitall() { # PID...
16 
+  ## Wait for children to exit and indicate whether all exited with 0 status.
17 
+  local errors=0
18 
+  while :; do
19 
+    #debug "Processes remaining: $*"
20 
+   DL_SIZE=0
21 
+   for i in $(seq 1 1 $MAX_SEGMENTS)
22 
+   do
23 
+      if [ -e "$FILENAME.part$i" ]
24 
+      then
25 
+	 SEG_SIZE=$(du -s $FILENAME.part$i |awk '{print $1}')
26 
+         DL_SIZE="$DL_SIZE+$SEG_SIZE"
27 
+      fi
28 
+   done
29 
+   CUR=$(echo "$DL_SIZE"|bc 2> /dev/null)
30 
+   OLD_PERCENT=$PERCENT
31 
+   PERCENT=$(echo "100*$CUR/($FILESIZE/1024)"|bc 2> /dev/null)
32 
+   if [ "$OLD_PERCENT" != "$PERCENT" ]
33 
+   then
34 
+      if [ "" != "$PERCENT" ]
35 
+      then
36 
+	 CUR_DATE=$(date +%s)
37 
+	 ELAPSED_TIME=$(($CUR_DATE - $DATE_START))
38 
+	 COMPLETE_TIME=$(echo "100*$ELAPSED_TIME/$PERCENT"|bc 2> /dev/null)
39 
+	 ETA=$(($COMPLETE_TIME - $ELAPSED_TIME))
40 
+	 ETA_HUMAN_MIN=$(echo "$ETA/60" | bc 2> /dev/null)
41 
+	 ETA_HUMAN_SEC=$(echo "$ETA - 60*$ETA_HUMAN_MIN" | bc 2> /dev/null)
42 
+	 AVG_SPEED=$(echo "$FILESIZE/1024/$COMPLETE_TIME" | bc 2> /dev/null)
43 
+         echo -ne "  $PERCENT % done ( ETA : $ETA_HUMAN_MIN min $(printf "%02d" $ETA_HUMAN_SEC) sec, AVG SPEED : $AVG_SPEED KBps )\r"
44 
+         #echo "$PERCENT % done ( ETA : $ETA_HUMAN_MIN min $(printf "%02d" $ETA_HUMAN_SEC) sec, AVG SPEED : $AVG_SPEED KBps )"
45 
+      fi
46 
+   fi
47 
+    for pid in "$@"; do
48 
+      shift
49 
+      if kill -0 "$pid" 2>/dev/null; then
50 
+        #debug "$pid is still alive."
51 
+        set -- "$@" "$pid"
52 
+      elif wait "$pid"; then
53 
+        #debug "$pid exited with zero exit status."
54 
+	echo "Segment downloaded successfully ($pid)"
55 
+      else
56 
+        debug "$pid exited with non-zero exit status."
57 
+        ((++errors))
58 
+      fi
59 
+    done
60 
+    (("$#" > 0)) || break
61 
+    # TODO: how to interrupt this sleep when a child terminates?
62 
+    sleep ${WAITALL_DELAY:-1}
63 
+   done
64 
+  ((errors == 0))
65 
+}
66 
+
67 
+
68 
+URL=$1
69 
+FILENAME=`basename $URL`
70 
+
71 
+echo "Downloading ${URL}"
72 
+
73 
+FILESIZE_TXT="./FILESIZE.txt"
74 
+curl -k -s -I ${URL} > ${FILESIZE_TXT}
75 
+i="0"
76 
+
77 
+FILESIZE="0"
78 
+while read myline
79 
+do
80 
+	if [[ "$myline" == Content-Length* ]]
81 
+	then
82 
+		#echo $i $myline
83 
+		FILESIZE=${myline:16}
84 
+		#echo ".${FILESIZE}."
85 
+ 		FILESIZE=${FILESIZE//[^0-9]/}
86 
+		#echo ".${FILESIZE}."
87 
+	fi
88 
+	i=`expr $((${i} + 1))`
89 
+done < ${FILESIZE_TXT}
90 
+
91 
+rm -f ${FILESIZE_TXT}
92 
+
93 
+#TODO Add exit at this point if this previous step fails.
94 
+
95 
+echo "Filesize is ${FILESIZE} bytes"
96 
+DATE_START=$(date +%s)
97 
+
98 
+SEGMENT_SIZE=`expr $((${FILESIZE} / ${MAX_SEGMENTS}))`
99 
+
100 
+echo "Segment size set to ${SEGMENT_SIZE} bytes"
101 
+
102 
+START_SEG=0;
103 
+END_SEG=${SEGMENT_SIZE}
104 
+pids=""
105 
+for i in $(eval echo {1..${MAX_SEGMENTS}})
106 
+do
107 
+	if [ "$i" !=  "$MAX_SEGMENT" ]
108 
+	then
109 
+		curl -k -s --range ${START_SEG}-${END_SEG} -o ${FILENAME}.part${i} ${URL} &
110 
+		pids="$pids $!"
111 
+	else
112 
+		curl -k -s --range ${START_SEG}- -o ${FILENAME}.part${i} ${URL} &
113 
+		pids="$pids $!"
114 
+	fi
115 
+	#echo "Start = ${START_SEG}"
116 
+	#echo "End = ${END_SEG}"
117 
+	START_SEG=$((${END_SEG}+1))
118 
+	END_SEG=$((${START_SEG}+${SEGMENT_SIZE}))
119 
+done
120 
+        echo "Starting to download file with $MAX_SEGMENTS thread(s)"
121 
+	waitall $pids
122 
+	echo "Download of segments is completed"
123 
+
124 
+
125 
+i=1
126 
+for i in $(eval echo {1..${MAX_SEGMENTS}})
127 
+do
128 
+	if [ "$i" == 1 ]
129 
+	then
130 
+		#echo "building segment:${i}"
131 
+		cat ${FILENAME}.part${i} > ${FILENAME}
132 
+	else
133 
+		#echo "building segment:${i}"
134 
+		cat ${FILENAME}.part${i} >> ${FILENAME}
135 
+	fi
136 
+done
137 
+for i in $(eval echo {1..${MAX_SEGMENTS}})
138 
+do
139 
+	rm -f ${FILENAME}.part${i}
140 
+done
141 
+echo "Download of ${FILENAME} is complete"
142 
+DATE_STOP=$(date +%s)
143 
+DELTA_TIME=$(($DATE_STOP-$DATE_START))
144 
+AVG_DL_SPEED=$(echo "($FILESIZE/$DELTA_TIME)/1024"| bc )
145 
+echo "Duration $DELTA_TIME"
146 
+if [ "" != "$2" ]
147 
+then
148 
+	mv ${FILENAME} $2
149 
+fi
150 
+echo "Average Speed $AVG_DL_SPEED KBps"
Gitprep Version v2.6.2 build on Mojolicious