2 Replies Latest reply on Dec 6, 2017 3:16 PM by Gael Hofemeier

    Why is this?

    Tony-BUG

      Hello, I would like to ask, how is this problem caused by me?

       

       

       

      2017-11-29 04:23:42.094725: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations.

      2017-11-29 04:23:42.094754: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations.

      2017-11-29 04:23:42.094763: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations.

      2017-11-29 04:23:42.094771: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations.

      2017-11-29 04:23:42.094779: W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations.

      terminate called after throwing an instance of 'std::system_error'

        what():  Resource temporarily unavailable

      Aborted

       

      My code  is follow:

       

      from datetime import datetime

      import math

      import time

      import tensorflow as tf

       

      batch_size = 32

      number_batches = 100

       

       

      def print_activation(t):

          print(t.op.name,' ',t.get_shape().as_list())

       

       

      def inference(images):

          parameters = []

       

       

          with tf.name_scope('conv1') as scope:

              kernel = tf.Variable(tf.truncated_normal([11,11,3,64],

                                                       dtype = tf.float32,stddev= 1e-1),name = 'weights')

              conv = tf.nn.conv2d(images,kernel,[1,4,4,1],padding = 'SAME')

              biases = tf.Variable(tf.constant(0.0,shape=[64],dtype = tf.float32),trainable = True ,name = "biases")

              bias = tf.nn.bias_add(conv,biases)

              conv1 = tf.nn.relu(bias,name = scope)

              print_activation(conv1)

              parameters += [kernel,biases]

       

       

              lrn1 = tf.nn.lrn(conv1,4,bias=1.0,alpha=0.001/9,beta= 0.75,name = 'lrn1')

              pool1 = tf.nn.max_pool(lrn1,ksize=[1,3,3,1],strides=[1,2,2,1],padding= "VALID",name='Pool1')

              print_activation(pool1)

       

       

          with tf.name_scope("conv2") as scope:

              kernel = tf.Variable(tf.truncated_normal([5,5,64,192],dtype = tf.float32,stddev = 1e-1),name = 'weights')

              conv = tf.nn.conv2d(pool1,kernel,[1,1,1,1],padding = 'SAME')

              biases = tf.Variable(tf.constant(0.0,shape = [192],dtype = tf.float32 ),trainable= True ,name = 'biases')

              bias = tf.nn.bias_add(conv,biases)

              conv2 = tf.nn.relu(bias,name = scope)

              parameters += [kernel,biases]

              print_activation(conv2)

       

       

              lrn2 = tf.nn.lrn(conv2,bias= 1.0,alpha=0.001/9,beta = 0.75, name='lrn2' )

              pool2 = tf.nn.max_pool(lrn2,ksize = [1,3,3,1],strides = [1,2,2,1],padding="VALID",name = 'pool2')

              print_activation(pool2)

       

       

          with tf.name_scope("conv3") as  scope:

              kernel = tf.Variable(tf.truncated_normal([3,3,192,384],dtype =tf.float32,stddev = 1e-1),name="weights")

              conv = tf.nn.conv2d(pool2,kernel,[1,1,1,1],padding= "SAME")

              biases = tf.Variable(tf.constant(0.0,shape=[384],dtype = tf.float32),trainable=True,name="biases")

              bias = tf.nn.bias_add(conv,biases)

              conv3 = tf.nn.relu(bias,name = scope)

              print_activation(conv3)

              parameters += [kernel,biases]

       

       

          with tf.name_scope("conv4") as scope:

              kernel = tf.Variable(tf.truncated_normal([3,3,384,256],dtype=tf.float32,stddev=1e-1),name= 'weights')

              conv = tf.nn.conv2d(conv3,kernel,[1,1,1,1],padding="SAME")

              biases = tf.Variable(tf.constant(0.0,shape=[256],dtype=tf.float32),trainable=True ,name= "biases")

              bias = tf.nn.bias_add(conv,biases)

              conv4 = tf.nn.relu(bias,name = scope)

              parameters += [kernel,biases]

              print_activation(conv4)

       

       

          with tf.name_scope("conv5") as scope:

              kernel = tf.Variable(tf.truncated_normal([3,3,256,256],dtype=tf.float32,stddev=1e-1),name= 'weights')

              conv = tf.nn.conv2d(conv4,kernel,[1,1,1,1],padding="SAME")

              biases = tf.Variable(tf.constant(0.0,shape=[256],dtype=tf.float32),trainable=True ,name= "biases")

              bias = tf.nn.bias_add(conv,biases)

              conv5 = tf.nn.relu(bias,name = scope)

              parameters += [kernel,biases]

              print_activation(conv5)

       

       

              # lrn5 = tf.nn.lrn(conv5,bias=1.0,alpha=0.001/9,beta=0.75,name='lrn5')

              pool5 = tf.nn.max_pool(conv5,[1,3,3,1],strides=[1,2,2,1],padding= "VALID",name='pool5' )

              print_activation(pool5)

       

       

          return pool5,parameters

       

       

      def time_tensorflow_run(session,target,info_string):

          num_step_burn_in = 10

          total_duration = 0.0

          total_duration_squared = 0.0

       

       

          for i in range(number_batches + num_step_burn_in):

              start_time = time.time()

              _ = session.run(target)

              duration  = time.time() - start_time

              if i >= num_step_burn_in:

                  if not i % 10 :

                      print ("%s:step %d,duration = %.3f"%(datetime.now(),i - num_step_burn_in,duration) )

                      total_duration += duration

                      total_duration_squared += duration * duration

       

       

          mn = total_duration /number_batches

          vr = total_duration_squared /number_batches - mn*mn

          sd = math.sqrt(vr)

          print("%s: %s across %d steps,%.3f +/- %.3f sec/batch"

                %(datetime.now(),info_string,number_batches,mn,sd))

       

       

      def run_benchmark():

          with tf.Graph().as_default():

              image_size = 224

              image = tf.Variable(tf.random_normal([batch_size,

                                                   image_size,

                                                   image_size,3],

                                                   dtype = tf.float32,

                                                   stddev = 1e-1))

              pool5 ,paramenters = inference(image)

       

       

              init = tf.global_variables_initializer()

       

       

              config = tf.ConfigProto()

              config.gpu_options.allocator_type = 'BFC'

              sess = tf.Session(config=config)

              sess.run(init)

       

       

              # sess = tf.Session

              # sess.run(init)

              time_tensorflow_run(sess,pool5,'Forward')

       

       

              objective = tf.nn.l2_loss(pool5)

              gard = tf.gradients(objective,paramenters)

              time_tensorflow_run(sess,gard,"Forward-backward")

       

       

       

       

      run_benchmark()

        • 1. Re: Why is this?
          Ravikeron

          Hello,

                Which environment you using to run your job. Is it DevCloud?

               If it is DevCloud, Are you using qsub command to run your job?  If not please use it.

               If you are already using qsub command, please try with setting the following options in the code.

              

          os.environ["OMP_NUM_THREADS"] = "64"

          os.environ["KMP_BLOCKTIME"] = "0"

          os.environ["KMP_SETTINGS"] = "1"

          os.environ["KMP_AFFINITY"]= "granularity=fine,verbose,compact,1,0"

           

          FLAGS = tf.app.flags.FLAGS

           

          tf.app.flags.DEFINE_integer('inter_op', 1, """Inter Op Parallelism Threads.""")

          tf.app.flags.DEFINE_integer('intra_op', 64, """Intra Op Parallelism Threads.""")

           

          Thanks

          Ravi Keron N

          • 2. Re: Why is this?
            Gael Hofemeier

            Did your issue get resolved?  We will need to close this if we do not receive any more updates.  You will be able to create a new issue if you are still facing difficulties.