Skip to content
This repository was archived by the owner on Nov 16, 2019. It is now read-only.

Commit 1c59767

Browse files
author
Mridul Jain
committed
Reduce logging
1 parent 9147590 commit 1c59767

File tree

5 files changed

+13
-17
lines changed

5 files changed

+13
-17
lines changed

‎Makefile‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
HOME ?=/home/${USER}
22
ifeq ($(shell which spark-submit),)
3-
SPARK_HOME ?=/home/y/share/spark
3+
SPARK_HOME ?= /home/y/share/spark
44
else
55
SPARK_HOME ?=$(shell which spark-submit 2>&1 | sed 's/\/bin\/spark-submit//g')
66
endif
77
CAFFE_ON_SPARK ?=$(shell pwd)
88
LD_LIBRARY_PATH ?=/home/y/lib64:/home/y/lib64/mkl/intel64:/usr/local/cuda/
99
LD_LIBRARY_PATH2=${LD_LIBRARY_PATH}:${CAFFE_ON_SPARK}/caffe-public/distribute/lib:${CAFFE_ON_SPARK}/caffe-distri/distribute/lib:/usr/lib64:/lib64
10-
DYLD_LIBRARY_PATH ?=/usr/local/cuda/lib
10+
DYLD_LIBRARY_PATH ?=/home/y/lib64:/home/y/lib64/mkl/intel64:/usr/local/cuda/lib
1111
DYLD_LIBRARY_PATH2=${DYLD_LIBRARY_PATH}:${CAFFE_ON_SPARK}/caffe-public/distribute/lib:${CAFFE_ON_SPARK}/caffe-distri/distribute/lib:/usr/lib64:/lib64
1212

1313
export SPARK_VERSION=$(shell ${SPARK_HOME}/bin/spark-submit --version 2>&1 | grep version | awk '{print $$5}' | cut -d'.' -f1)
@@ -17,21 +17,21 @@ endif
1717

1818
build:
1919
cd caffe-public; make proto; make -j4 -e distribute; cd ..
20-
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH2}"; mvn ${MVN_SPARK_FLAG} -B package -DskipTests
20+
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH2}"; GLOG_minloglevel=1 mvn ${MVN_SPARK_FLAG} -B package -DskipTests
2121
jar -xvf caffe-grid/target/caffe-grid-0.1-SNAPSHOT-jar-with-dependencies.jar META-INF/native/linux64/liblmdbjni.so
2222
mv META-INF/native/linux64/liblmdbjni.so ${CAFFE_ON_SPARK}/caffe-distri/distribute/lib
2323
${CAFFE_ON_SPARK}/scripts/setup-mnist.sh
24-
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH2}"; mvn ${MVN_SPARK_FLAG} -B test
24+
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH2}"; GLOG_minloglevel=1 mvn ${MVN_SPARK_FLAG} -B test
2525
cd ${CAFFE_ON_SPARK}/caffe-grid/src/main/python/; zip -r caffeonsparkpythonapi *; cd ${CAFFE_ON_SPARK}/caffe-public/python/; zip -ur ${CAFFE_ON_SPARK}/caffe-grid/src/main/python/caffeonsparkpythonapi.zip *; cd - ; mv caffeonsparkpythonapi.zip ${CAFFE_ON_SPARK}/caffe-grid/target/; cd ${CAFFE_ON_SPARK}
2626
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}; export SPARK_HOME=${SPARK_HOME};${CAFFE_ON_SPARK}/caffe-grid/src/test/python/PythonTest.sh
2727

2828
buildosx:
2929
cd caffe-public; make proto; make -j4 -e distribute; cd ..
30-
export DYLD_LIBRARY_PATH="${DYLD_LIBRARY_PATH2}"; mvn ${MVN_SPARK_FLAG} -B package -DskipTests
30+
export DYLD_LIBRARY_PATH="${DYLD_LIBRARY_PATH2}"; GLOG_minloglevel=1 mvn ${MVN_SPARK_FLAG} -B package -DskipTests
3131
jar -xvf caffe-grid/target/caffe-grid-0.1-SNAPSHOT-jar-with-dependencies.jar META-INF/native/osx64/liblmdbjni.jnilib
3232
mv META-INF/native/osx64/liblmdbjni.jnilib ${CAFFE_ON_SPARK}/caffe-distri/distribute/lib
3333
${CAFFE_ON_SPARK}/scripts/setup-mnist.sh
34-
export LD_LIBRARY_PATH="${DYLD_LIBRARY_PATH2}"; mvn ${MVN_SPARK_FLAG} -B test
34+
export LD_LIBRARY_PATH="${DYLD_LIBRARY_PATH2}"; GLOG_minloglevel=1 mvn ${MVN_SPARK_FLAG} -B test
3535
cd ${CAFFE_ON_SPARK}/caffe-grid/src/main/python/; zip -r caffeonsparkpythonapi *; cd ${CAFFE_ON_SPARK}/caffe-public/python/; zip -ur ${CAFFE_ON_SPARK}/caffe-grid/src/main/python/caffeonsparkpythonapi.zip *; cd -; mv caffeonsparkpythonapi.zip ${CAFFE_ON_SPARK}/caffe-grid/target/; cd ${CAFFE_ON_SPARK}
3636
cd ${CAFFE_ON_SPARK}/caffe-grid/src/main/python/; zip -r caffeonsparkpythonapi *; mv caffeonsparkpythonapi.zip ${CAFFE_ON_SPARK}/caffe-grid/target/; cd ${CAFFE_ON_SPARK}
3737
export DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}; export SPARK_HOME=${SPARK_HOME};${CAFFE_ON_SPARK}/caffe-grid/src/test/python/PythonTest.sh

‎caffe-grid/src/main/python/com/yahoo/ml/caffe/DisplayUtils.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import numpy as np
55
from base64 import b64encode
66
from google.protobuf import text_format
7-
import array
87

98
import caffe
109
import caffe.draw

‎caffe-grid/src/main/python/examples/README-coco.md‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,14 @@
11
Steps to run the COCO dataset for Image Captioning
22
==================================================
33
##### (1) Env setup
4-
Initial Setup: https://github.com/yahoo/CaffeOnSpark/wiki/GetStarted_standalone
4+
Set up both CAFFE_ON_SPARK and SPARK_HOME per https://github.com/yahoo/CaffeOnSpark/wiki/GetStarted_standalone
55
export DYLD_LIBRARY_PATH=${CAFFE_ON_SPARK}/caffe-public/distribute/lib:${CAFFE_ON_SPARK}/caffe-distri/distribute/lib:/usr/local/cuda/lib:/usr/local/mkl/lib/intel64/:Python2.7.10/lib:/usr/local/cuda/lib:caffe_on_grid_archive/lib64/mkl/intel64/
66
export LD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}
77
export SPARK_HOME=/Users/mridul/bigml/spark-1.6.0-bin-hadoop2.6
88
export PATH=${SPARK_HOME}/bin:${PATH}
99
export PYSPARK_PYTHON=Python2.7.10/bin/python
1010
export PYTHONPATH=$PYTHONPATH:caffeonsparkpythonapi.zip:caffe_on_grid_archive/lib64:/usr/local/cuda-7.5/lib64
1111
export IPYTHON_ROOT=~/Python2.7.10
12-
unset SPARK_CONF_DIR
1312

1413
##### (2) Download the coco dataset if required
1514

@@ -81,7 +80,8 @@ Steps to run the COCO dataset for Image Captioning
8180
-model /tmp/coco/parquet/lrcn_coco.model
8281
popd
8382

84-
##### (6) Submit the data for inference
83+
#### Run either of the steps below for running a script or notebook
84+
##### (6 a) Submit the data for inference
8585
Note that the below files also need to be shipped as shown
8686
#-model the image-lstm pretrained model to ship
8787
#-imagenet the image network definition
@@ -93,25 +93,23 @@ Steps to run the COCO dataset for Image Captioning
9393
pushd ${CAFFE_ON_SPARK}/data/
9494
ln -s ~/Python2.7.10 Python2.7.10
9595
unzip ${CAFFE_ON_SPARK}/caffe-grid/target/caffeonsparkpythonapi.zip
96-
cat /tmp/coco/parquet/vocab/part* > vocab.txt
9796
rm -rf /tmp/coco/parquet/df_caption_results_train2014
9897
spark-submit --master ${MASTER_URL} \
9998
--conf spark.cores.max=${TOTAL_CORES} \
10099
--conf spark.task.cpus=${CORES_PER_WORKER} \
101100
--conf spark.driver.extraLibraryPath="${DYLD_LIBRARY_PATH}:Python2.7.10/lib" \
102101
--conf spark.executorEnv.LD_LIBRARY_PATH="${DYLD_LIBRARY_PATH}:Python2.7.10/lib" \
103-
--conf spark.pythonargs="-model /tmp/coco/parquet/lrcn_coco.model -imagenet lstm_deploy.prototxt -lstmnet lrcn_word_to_preds.deploy.prototxt -vocab vocab.txt -input /tmp/coco/parquet/df_embedded_train2014 -output /tmp/coco/parquet/df_caption_results_train2014" examples/ImageCaption.py
102+
--conf spark.pythonargs="-model /tmp/coco/parquet/lrcn_coco.model -imagenet lstm_deploy.prototxt -lstmnet lrcn_word_to_preds.deploy.prototxt -vocab /tmp/coco/parquet/vocab/part-00000 -input /tmp/coco/parquet/df_embedded_train2014 -output /tmp/coco/parquet/df_caption_results_train2014" examples/ImageCaption.py
104103
popd
105-
##### (7) Launch IPython Notebook
104+
##### (6 b) Launch IPython Notebook
106105
export IPYTHON_OPTS="notebook --no-browser --ip=127.0.0.1"
107106
pushd ${CAFFE_ON_SPARK}/data/
108107
ln -s ~/Python2.7.10 Python2.7.10
109108
unzip ${CAFFE_ON_SPARK}/caffe-grid/target/caffeonsparkpythonapi.zip
110-
cat /tmp/coco/parquet/vocab/part* > vocab.txt
111109
pyspark --master ${MASTER_URL} --deploy-mode client \
112110
--conf spark.driver.extraLibraryPath="${DYLD_LIBRARY_PATH}:Python2.7.10/lib" \
113111
--conf spark.executorEnv.LD_LIBRARY_PATH="${DYLD_LIBRARY_PATH}:Python2.7.10/lib" \
114-
--files "${CAFFE_ON_SPARK}/data/lstm_deploy.prototxt,${CAFFE_ON_SPARK}/data/vocab.txt,${CAFFE_ON_SPARK}/data/lrcn_word_to_preds.deploy.prototxt,${CAFFE_ON_SPARK}/data/caffe/_caffe.so,${CAFFE_ON_SPARK}/data/bvlc_reference_net.prototxt,${CAFFE_ON_SPARK}/data/bvlc_reference_solver.prototxt,${CAFFE_ON_SPARK}/data/lrcn_cos.prototxt,${CAFFE_ON_SPARK}/data/lrcn_solver.prototxt" \
112+
--files "${CAFFE_ON_SPARK}/data/lstm_deploy.prototxt,${CAFFE_ON_SPARK}/data/vocab.txt/part-00000,${CAFFE_ON_SPARK}/data/lrcn_word_to_preds.deploy.prototxt,${CAFFE_ON_SPARK}/data/caffe/_caffe.so,${CAFFE_ON_SPARK}/data/bvlc_reference_net.prototxt,${CAFFE_ON_SPARK}/data/bvlc_reference_solver.prototxt,${CAFFE_ON_SPARK}/data/lrcn_cos.prototxt,${CAFFE_ON_SPARK}/data/lrcn_solver.prototxt" \
115113
--py-files "${CAFFE_ON_SPARK}/caffe-grid/target/caffeonsparkpythonapi.zip" \
116114
--jars "${CAFFE_ON_SPARK}/caffe-grid/target/caffe-grid-0.1-SNAPSHOT-jar-with-dependencies.jar" \
117115
--driver-library-path "${CAFFE_ON_SPARK}/caffe-grid/target/caffe-grid-0.1-SNAPSHOT-jar-with-dependencies.jar" \

‎caffe-grid/src/main/scala/com/yahoo/ml/caffe/tools/Vocab.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ private[tools] class Vocab(val sqlContext: SQLContext) {
4444

4545
def save(vocabFilePath: String): Unit = {
4646
synchronized {
47-
rdd_word.map(word => word.getString(0)).saveAsTextFile(vocabFilePath)
47+
rdd_word.map(word => word.getString(0)).coalesce(1, true).saveAsTextFile(vocabFilePath)
4848
}
4949
}
5050

‎caffe-grid/src/test/scala/com/yahoo/ml/caffe/tools/ToolTest.scala‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,6 @@ class ToolTest extends FunSuite with BeforeAndAfterAll {
9696

9797
FileUtils.deleteQuietly(new File(cocoImageCaptionDF))
9898
val df_image_caption = Conversions.Coco2ImageCaptionFile(sqlContext, cocoJson, 4)
99-
// val rdd_input_captions = inputDF2PairRDD(df_image_caption)
10099
val count = df_image_caption.count.toInt
101100

102101
FileUtils.deleteQuietly(new File(cocoEmbeddingDF))

0 commit comments

Comments
 (0)