Tensor flow on Flink taking too much time to process data
- Dominant language
- Java
- Stars
- 8.7k
- Forks
- 4.7k
- Avg merge
- 1d 20h
- Merged PRs (30d)
- 196
Description
The TFX pipeline is taking too much time to process data. The SDK is looping in the following stacktrace. Also, the CPU usage is 400% which is in alignment with 4 threads that are working.
```
--- Thread #140477558937344 name: Thread-55 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 74,
in _worker work_item.run() File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 62, in run result = self.fn(*self.args, **self.kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in task self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 132, in _execute response = task() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 219, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 234, in process_bundle bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 437, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 125, in process_encoded self.output(decoded_value) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py",
line 1448, in process *args, **kwargs)) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py",
line 587, in add_inputs accumulator = self.add_input(accumulator, element, *args, **kwargs) File "/usr/local/lib/python2.7/site-packages/tensorflow_transform/beam/analyzer_impls.py",
line 264, in add_input summary=tf.constant(summary))) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 905, in run run_metadata_ptr) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1137, in _run feed_dict_tensor, options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1355, in _do_run options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1361, in _do_call return fn(*args) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1332, in _run_fn self._extend_graph() File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1392, in _extend_graph graph_def.SerializeToString(), status)
```
Complete stack trace
```
--- Thread #140471896581888 name: Thread-153 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140480218113792 name: Thread-12 --- File "/usr/local/lib/python2.7/threading.py", line 774,
in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140471528552192 name: Thread-199 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140476451624704 name: Thread-86 --- File "/usr/local/lib/python2.7/threading.py", line 774,
in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140471930152704 name: Thread-143 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140471740630784 name: Thread-183 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140479144371968 name: Thread-40 --- File "/usr/local/lib/python2.7/threading.py", line 774,
in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140479152764672 name: Thread-34 --- File "/usr/local/lib/python2.7/threading.py", line 774,
in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140472399947520 name: Thread-122 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140475956717312 name: Thread-88 --- File "/usr/local/lib/python2.7/threading.py", line 774,
in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140479689635584 name: Thread-15 --- File "/usr/local/lib/python2.7/threading.py", line 774,
in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140471804360448 name: Thread-150 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140472391554816 name: Thread-126 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140474602440448 name: Thread-214 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140480226506496 name: Thread-6 --- File "/usr/local/lib/python2.7/threading.py", line 774,
in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 191, in consume_request_iterator
request = next(request_iterator) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/log_handler.py",
line 111, in _write_log_entries log_entries = [self._log_entry_queue.get()] File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140476501980928 name: Thread-98 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471731975936 name: Thread-162 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140476996888320 name: Thread-92 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471692900096 name: Thread-167 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140472383162112 name: Thread-128 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140477038851840 name: Thread-74 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140477533759232 name: Thread-120 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140477558937344 name: Thread-55 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 74, in _worker work_item.run() File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 62, in run result = self.fn(*self.args, **self.kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in task self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 132, in _execute response = task() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 219, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 234, in process_bundle bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 437, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 125, in process_encoded self.output(decoded_value) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py",
line 1448, in process *args, **kwargs)) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py",
line 587, in add_inputs accumulator = self.add_input(accumulator, element, *args, **kwargs) File "/usr/local/lib/python2.7/site-packages/tensorflow_transform/beam/analyzer_impls.py",
line 264, in add_input summary=tf.constant(summary))) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 905, in run run_metadata_ptr) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1137, in _run feed_dict_tensor, options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1355, in _do_run options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1361, in _do_call return fn(*args) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1332, in _run_fn self._extend_graph() File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1392, in _extend_graph graph_def.SerializeToString(), status) --- Thread #140477575722752 name:
Thread-48 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner()
File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py",
line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140472374769408 name: Thread-129 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140478070630144 name: Thread-52 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471603033856 name: Thread-181 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140478112593664 name: Thread-41 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471563171584 name: Thread-178 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140478607501056 name: Thread-29 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140472366376704 name: Thread-130 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471493146368 name: Thread-220 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140474594047744 name: Thread-208 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140472357984000 name: Thread-138 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) --- Thread #140471904974592 name: Thread-146 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140472005687040 name: Thread-137 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140479681242880 name: Thread-16 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471795967744 name: Thread-151 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471749023488 name: Thread-176 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140479723206400 name: Thread-11 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 191, in consume_request_iterator request = next(request_iterator) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 104, in get_responses response = self._responses.get() File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471972116224 name: Thread-136 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471997294336 name: Thread-133 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140476988495616 name: Thread-76 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471783118592 name: Thread-155 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471723583232 name: Thread-168 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140479186335488 name: Thread-18 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 191, in consume_request_iterator request = next(request_iterator) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 244, in _write_outputs data = [self._to_send.get()] File "/usr/local/lib/python2.7/Queue.py", line
168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140477030459136 name: Thread-68 --- File "/usr/local/lib/python2.7/threading.py", line 774,
in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140471510718208 name: Thread-191 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140471988901632 name: Thread-134 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) --- Thread #140476485195520 name: Thread-80 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 74, in _worker work_item.run() File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 62, in run result = self.fn(*self.args, **self.kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in task self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 132, in _execute response = task() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 219, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 234, in process_bundle bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 437, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 125, in process_encoded self.output(decoded_value) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py",
line 1448, in process *args, **kwargs)) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py",
line 587, in add_inputs accumulator = self.add_input(accumulator, element, *args, **kwargs) File "/usr/local/lib/python2.7/site-packages/tensorflow_transform/beam/analyzer_impls.py",
line 275, in add_input next_stamp_token=self._stamp_token)) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 905, in run run_metadata_ptr) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1137, in _run feed_dict_tensor, options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1355, in _do_run options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1361, in _do_call return fn(*args) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1332, in _run_fn self._extend_graph() File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1392, in _extend_graph graph_def.SerializeToString(), status) --- Thread #140477567330048 name:
Thread-59 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner()
File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py",
line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140479698028288 name: read_state --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 390, in pull_responses for response in responses: File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 366, in next return self._next() File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 348, in _next self._state.condition.wait() File "/usr/local/lib/python2.7/threading.py", line 340,
in wait waiter.acquire() --- Thread #140478062237440 name: Thread-63 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471980508928 name: Thread-135 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140474220279552 name: Thread-114 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) --- Thread #140478104200960 name: Thread-37 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471484229376 name: Thread-209 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140478599108352 name: Thread-31 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471938545408 name: Thread-142 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140478641071872 name: Thread-25 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471701554944 name: Thread-170 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140479135979264 name: Thread-50 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140479177942784 name: read_grpc_client_inputs --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 273, in target=lambda: self._read_inputs(elements_iterator), File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 260, in _read_inputs for elements in elements_iterator: File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 366, in next return self._next() File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 348, in _next self._state.condition.wait() File "/usr/local/lib/python2.7/threading.py", line 340,
in wait waiter.acquire() --- Thread #140474568869632 name: Thread-115 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140475948324608 name: Thread-105 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140474203494144 name: Thread-117 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140481212552960 name: MainThread --- File "/usr/local/lib/python2.7/runpy.py",
line 174, in _run_module_as_main "__main__", fname, loader, pkg_name) File "/usr/local/lib/python2.7/runpy.py",
line 72, in _run_code exec code in run_globals File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker_main.py",
line 213, in main(sys.argv) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker_main.py",
line 141, in main worker_count=_get_worker_count(sdk_pipeline_options)).run() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 109, in run for work_request in control_stub.Control(get_responses()): File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 366, in next return self._next() File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 348, in _next self._state.condition.wait() File "/usr/local/lib/python2.7/threading.py", line 340,
in wait waiter.acquire() --- Thread #140471913367296 name: Thread-152 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140480209721088 name: Thread-10 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 703, in channel_spin event = state.channel.next_call_event() --- Thread #140471955330816 name:
Thread-140 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner()
File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py",
line 754, in run self.__target(*self.__args, **self.__kwargs) --- Thread #140474195101440 name: Thread-212
--- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner()
File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py",
line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140478079022848 name: Thread-44 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 74, in _worker work_item.run() File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 62, in run result = self.fn(*self.args, **self.kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in task self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 132, in _execute response = task() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 219, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 234, in process_bundle bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 437, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 125, in process_encoded self.output(decoded_value) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py",
line 1448, in process *args, **kwargs)) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py",
line 587, in add_inputs accumulator = self.add_input(accumulator, element, *args, **kwargs) File "/usr/local/lib/python2.7/site-packages/tensorflow_transform/beam/analyzer_impls.py",
line 275, in add_input next_stamp_token=self._stamp_token)) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 905, in run run_metadata_ptr) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1137, in _run feed_dict_tensor, options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1355, in _do_run options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1361, in _do_call return fn(*args) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1332, in _run_fn self._extend_graph() File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1392, in _extend_graph graph_def.SerializeToString(), status) --- Thread #140471577855744 name:
Thread-180 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner()
File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py",
line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471946938112 name: Thread-216 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140477550544640 name: Thread-65 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 74, in _worker work_item.run() File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 62, in run result = self.fn(*self.args, **self.kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in task self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 132, in _execute response = task() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 219, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 234, in process_bundle bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 434, in process_bundle instruction_id, expected_targets): File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 205, in input_elements data = received.get(timeout=1) File "/usr/local/lib/python2.7/Queue.py",
line 177, in get self.not_empty.wait(remaining) File "/usr/local/lib/python2.7/threading.py", line 359,
in wait _sleep(delay) --- Thread #140471545337600 name: Thread-184 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471594641152 name: Thread-173 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140474186708736 name: Thread-124 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140478095808256 name: Thread-38 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140474585655040 name: Thread-206 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471774725888 name: Thread-157 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140478632679168 name: Thread-46 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471766071040 name: Thread-158 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) --- Thread #140480260077312 name: Thread-13 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 703, in channel_spin event = state.channel.next_call_event() --- Thread #140471519373056 name:
Thread-189 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner()
File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py",
line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140479169550080 name: Thread-19 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args,
issue truncated because of its length - to see full context, see original Jira
Imported from Jira [BEAM-6029](https://issues.apache.org/jira/browse/BEAM-6029). Original Jira may contain additional context.
Reported by: angoenka.
Contributor guide
Assessment
This issue has not been assessed yet.