diff --git a/composer.json b/composer.json index b8a816c0e2d2..6b520ef1b81d 100644 --- a/composer.json +++ b/composer.json @@ -85,6 +85,9 @@ "scripts": { "google-cloud": "dev/google-cloud" }, + "bin": [ + "src/Core/bin/google-cloud-batch" + ], "extra": { "component": { "id": "google-cloud", diff --git a/dev/sh/clear-ipc b/dev/sh/clear-ipc new file mode 100755 index 000000000000..7582a290f34f --- /dev/null +++ b/dev/sh/clear-ipc @@ -0,0 +1,7 @@ +#!/bin/bash + +set -ev + +ipcs -m | grep `whoami` | awk '{ print $2 }' | xargs -n1 ipcrm -m +ipcs -s | grep `whoami` | awk '{ print $2 }' | xargs -n1 ipcrm -s +ipcs -q | grep `whoami` | awk '{ print $2 }' | xargs -n1 ipcrm -q diff --git a/dev/src/DocGenerator/DocGenerator.php b/dev/src/DocGenerator/DocGenerator.php index 5d1736a96127..3b3874915ae6 100644 --- a/dev/src/DocGenerator/DocGenerator.php +++ b/dev/src/DocGenerator/DocGenerator.php @@ -97,17 +97,18 @@ public function generate($basePath, $pretty) } $document = $parser->parse(); + if ($document) { + $writer = new Writer($document, $this->outputPath, $pretty); + $writer->write($currentFile); - $writer = new Writer($document, $this->outputPath, $pretty); - $writer->write($currentFile); - - $this->types->addType([ - 'id' => $document['id'], - 'title' => $document['title'], - 'contents' => ($this->isComponent) - ? $this->prune($document['id'] . '.json') - : $document['id'] . '.json' - ]); + $this->types->addType([ + 'id' => $document['id'], + 'title' => $document['title'], + 'contents' => ($this->isComponent) + ? $this->prune($document['id'] . '.json') + : $document['id'] . '.json' + ]); + } } } diff --git a/dev/src/DocGenerator/Parser/CodeParser.php b/dev/src/DocGenerator/Parser/CodeParser.php index b8a4979e04e0..b9036b0e5519 100644 --- a/dev/src/DocGenerator/Parser/CodeParser.php +++ b/dev/src/DocGenerator/Parser/CodeParser.php @@ -69,7 +69,11 @@ public function __construct( public function parse() { $this->reflector->process(); - return $this->buildDocument($this->getReflector($this->reflector)); + $reflector = $this->getReflector($this->reflector); + + return $reflector + ? $this->buildDocument($reflector) + : null; } private function getReflector($fileReflector) @@ -86,7 +90,7 @@ private function getReflector($fileReflector) return $fileReflector->getTraits()[0]; } - throw new \Exception('Could not get reflector for '. $this->fileName); + return null; } private function buildDocument($reflector) diff --git a/dev/src/Snippet/Parser/Parser.php b/dev/src/Snippet/Parser/Parser.php index 06007b9817d1..6bcd49673970 100644 --- a/dev/src/Snippet/Parser/Parser.php +++ b/dev/src/Snippet/Parser/Parser.php @@ -137,14 +137,16 @@ public function examplesFromClass($class) $methods = $this->buildMagicMethods($magicMethods, $class->getName()); foreach ($methods as $method) { - $res = current($this->examples( + $magicExamples = $this->examples( $method['doc'], $class->getName() .'::'. $method['name'], $class->getFileName(), $class->getStartLine() - )); + ); - $magic[$res->identifier()] = $res; + foreach ($magicExamples as $ex) { + $magic[$ex->identifier()] = $ex; + } } } diff --git a/src/Core/Batch/BatchConfig.php b/src/Core/Batch/BatchConfig.php new file mode 100644 index 000000000000..663d0219a94a --- /dev/null +++ b/src/Core/Batch/BatchConfig.php @@ -0,0 +1,113 @@ +idmap) + ? $this->jobs[$identifier] + : null; + } + + /** + * Get the job with the given numeric id. + * + * @param int $idNum A numeric id of the job. + * + * @return BatchJob|null + */ + public function getJobFromIdNum($idNum) + { + return array_key_exists($idNum, $this->idmap_reverse) + ? $this->jobs[$this->idmap_reverse[$idNum]] + : null; + } + + /** + * Register a job for executing in batch. + * + * @param string $identifier Unique identifier of the job. + * @param callable $func Any Callable except for Closure. The callable + * should accept an array of items as the first argument. + * @param array $options [optional] { + * Configuration options. + * + * @type int $batchSize The size of the batch. + * @type float $callPeriod The period in seconds from the last execution + * to force executing the job. + * @type int $workerNum The number of child processes. It only takes + * effect with the {@see \Google\Cloud\Core\BatchDaemon}. + * @type string $bootstrapFile A file to load before executing the + * job. It's needed for registering global functions. + * } + * @return void + */ + public function registerJob($identifier, $func, array $options = []) + { + if (array_key_exists($identifier, $this->idmap)) { + $idNum = $this->idmap[$identifier]; + } else { + $idNum = count($this->idmap) + 1; + $this->idmap_reverse[$idNum] = $identifier; + } + $this->jobs[$identifier] = new BatchJob( + $identifier, + $func, + $idNum, + $options + ); + $this->idmap[$identifier] = $idNum; + } + + /** + * Get all the jobs. + * + * @return BatchJob[] + */ + public function getJobs() + { + return $this->jobs; + } +} diff --git a/src/Core/Batch/BatchDaemon.php b/src/Core/Batch/BatchDaemon.php new file mode 100644 index 000000000000..6c6d30944ea4 --- /dev/null +++ b/src/Core/Batch/BatchDaemon.php @@ -0,0 +1,227 @@ +isSysvIPCLoaded()) { + throw new \RuntimeException('SystemV IPC extensions are missing.'); + } + $this->runner = new BatchRunner( + new SysvConfigStorage(), + new SysvProcessor() + ); + $this->shutdown = false; + // Just share the usual descriptors. + $this->descriptorSpec = [ + 0 => ['file', 'php://stdin', 'r'], + 1 => ['file', 'php://stdout', 'w'], + 2 => ['file', 'php://stderr', 'w'] + ]; + // setup signal handlers + pcntl_signal(SIGTERM, [$this, "sigHandler"]); + pcntl_signal(SIGINT, [$this, "sigHandler"]); + pcntl_signal(SIGHUP, [$this, "sigHandler"]); + pcntl_signal(SIGALRM, [$this, "sigHandler"]); + $this->command = sprintf('exec php -d auto_prepend_file="" %s daemon', $entrypoint); + $this->initFailureFile(); + } + + /** + * A signal handler for setting the terminate switch. + * {@see http://php.net/manual/en/function.pcntl-signal.php} + * + * @param int $signo The received signal. + * @param mixed $siginfo [optional] An array representing the signal + * information. **Defaults to** null. + * + * @return void + */ + public function sigHandler($signo, $signinfo = null) + { + switch ($signo) { + case SIGINT: + case SIGTERM: + $this->shutdown = true; + break; + } + } + + /** + * A loop for the parent. + * + * @return void + */ + public function runParent() + { + $procs = []; + while (true) { + $jobs = $this->runner->getJobs(); + foreach ($jobs as $job) { + if (! array_key_exists($job->getIdentifier(), $procs)) { + $procs[$job->getIdentifier()] = []; + } + while (count($procs[$job->getIdentifier()]) > $job->getWorkerNum()) { + // Stopping an excessive child. + echo 'Stopping an excessive child.' . PHP_EOL; + $proc = array_pop($procs[$job->getIdentifier()]); + $status = proc_get_status($proc); + // Keep sending SIGTERM until the child exits. + while ($status['running'] === true) { + @proc_terminate($proc); + usleep(50000); + $status = proc_get_status($proc); + } + @proc_close($proc); + } + for ($i = 0; $i < $job->getWorkerNum(); $i++) { + $needStart = false; + if (array_key_exists($i, $procs[$job->getIdentifier()])) { + $status = proc_get_status($procs[$job->getIdentifier()][$i]); + if ($status['running'] !== true) { + $needStart = true; + } + } else { + $needStart = true; + } + if ($needStart) { + echo 'Starting a child.' . PHP_EOL; + $procs[$job->getIdentifier()][$i] = proc_open( + sprintf('%s %d', $this->command, $job->getIdNum()), + $this->descriptorSpec, + $pipes + ); + } + } + } + usleep(1000000); // Reload the config after 1 second + pcntl_signal_dispatch(); + if ($this->shutdown) { + echo 'Shutting down, waiting for the children' . PHP_EOL; + foreach ($procs as $k => $v) { + foreach ($v as $proc) { + $status = proc_get_status($proc); + // Keep sending SIGTERM until the child exits. + while ($status['running'] === true) { + @proc_terminate($proc); + usleep(50000); + $status = proc_get_status($proc); + } + @proc_close($proc); + } + } + echo 'BatchDaemon exiting' . PHP_EOL; + exit; + } + // Reload the config + $this->runner->loadConfig(); + } + } + + /** + * A loop for the children. + * + * @param int $idNum Numeric id for the job. + * @return void + */ + public function runChild($idNum) + { + // child process + $sysvKey = $this->getSysvKey($idNum); + $q = msg_get_queue($sysvKey); + $items = []; + $job = $this->runner->getJobFromIdNum($idNum); + $period = $job->getCallPeriod(); + $lastInvoked = microtime(true); + $batchSize = $job->getBatchSize(); + while (true) { + // Fire SIGALRM after 1 second to unblock the blocking call. + pcntl_alarm(1); + if (msg_receive( + $q, + 0, + $type, + 8192, + $message, + true, + 0, // blocking mode + $errorcode + )) { + if ($type === self::$typeDirect) { + $items[] = $message; + } elseif ($type === self::$typeFile) { + $items[] = unserialize(file_get_contents($message)); + @unlink($message); + } + } + pcntl_signal_dispatch(); + // It runs the job when + // 1. Number of items reaches the batchSize. + // 2-a. Count is >0 and the current time is larger than lastInvoked + period. + // 2-b. Count is >0 and the shutdown flag is true. + if ((count($items) >= $batchSize) + || (count($items) > 0 + && (microtime(true) > $lastInvoked + $period + || $this->shutdown))) { + printf( + 'Running the job with %d items' . PHP_EOL, + count($items) + ); + if (! $job->run($items)) { + $this->handleFailure($idNum, $items); + } + $items = []; + $lastInvoked = microtime(true); + } + gc_collect_cycles(); + if ($this->shutdown) { + exit; + } + } + } +} diff --git a/src/Core/Batch/BatchJob.php b/src/Core/Batch/BatchJob.php new file mode 100644 index 000000000000..60ad6dfc0179 --- /dev/null +++ b/src/Core/Batch/BatchJob.php @@ -0,0 +1,167 @@ +identifier = $identifier; + $this->func = $func; + $this->idNum = $idNum; + $this->batchSize = array_key_exists('batchSize', $options) + ? $options['batchSize'] + : self::DEFAULT_BATCH_SIZE; + $this->callPeriod = array_key_exists('callPeriod', $options) + ? $options['callPeriod'] + : self::DEFAULT_CALL_PERIOD; + $this->bootstrapFile = array_key_exists('bootstrapFile', $options) + ? $options['bootstrapFile'] + : null; + $this->workerNum = array_key_exists('workerNum', $options) + ? $options['workerNum'] + : self::DEFAULT_WORKERS; + } + + /** + * Run the job with the given items. + * + * @param array $items An array of items. + * + * @return bool the result of the callback + */ + public function run(array $items) + { + if (! is_null($this->bootstrapFile)) { + require_once($this->bootstrapFile); + } + return call_user_func_array($this->func, [$items]); + } + + /** + * @return int + */ + public function getIdNum() + { + return $this->idNum; + } + + /** + * @return string + */ + public function getIdentifier() + { + return $this->identifier; + } + + /** + * @return float + */ + public function getCallPeriod() + { + return $this->callPeriod; + } + + /** + * @return int + */ + public function getBatchSize() + { + return $this->batchSize; + } + + /** + * @return int + */ + public function getWorkerNum() + { + return $this->workerNum; + } + + /** + * @return string + */ + public function getBootstrapFile() + { + return $this->bootstrapFile; + } +} diff --git a/src/Core/Batch/BatchRunner.php b/src/Core/Batch/BatchRunner.php new file mode 100644 index 000000000000..33666ee87910 --- /dev/null +++ b/src/Core/Batch/BatchRunner.php @@ -0,0 +1,195 @@ +isSysvIPCLoaded() && $this->isDaemonRunning()) { + $configStorage = new SysvConfigStorage(); + $processor = new SysvProcessor(); + } else { + $configStorage = InMemoryConfigStorage::getInstance(); + $processor = $configStorage; + } + } + $this->configStorage = $configStorage; + $this->processor = $processor; + $this->loadConfig(); + } + + /** + * Register a job for batch execution. + * + * @param string $identifier Unique identifier of the job. + * @param callable $func Any Callable except for Closure. The callable + * should accept an array of items as the first argument. + * @param array $options [optional] { + * Configuration options. + * + * @type int $batchSize The size of the batch. + * @type float $callPeriod The period in seconds from the last execution + * to force executing the job. + * @type int $workerNum The number of child processes. It only takes + * effect with the {@see \Google\Cloud\Core\BatchDaemon}. + * @type string $bootstrapFile A file to load before executing the + * job. It's needed for registering global functions. + * } + * @return bool true on success, false on failure + * @throws \InvalidArgumentException When receiving a Closure. + */ + public function registerJob($identifier, $func, array $options = []) + { + if ($func instanceof \Closure) { + throw new \InvalidArgumentException('Closure is not allowed'); + } + // Always work on the latest data + $result = $this->configStorage->lock(); + if ($result === false) { + return false; + } + $this->config = $this->configStorage->load(); + $this->config->registerJob($identifier, $func, $options); + + $result = $this->configStorage->save($this->config); + if ($result === false) { + return false; + } + $this->configStorage->unlock(); + return true; + } + + /** + * Submit an item. + * + * @param string $identifier Unique identifier of the job. + * @param mixed $item It needs to be serializable. + * + * @return bool true on success, false on failure + * @throws \RuntimeException + */ + public function submitItem($identifier, $item) + { + $job = $this->getJobFromId($identifier); + if ($job === null) { + throw new \RuntimeException( + "The identifier does not exist: $identifier" + ); + } + $idNum = $job->getIdnum(); + return $this->processor->submit($item, $idNum); + } + + /** + * Get the job with the given identifier. + * + * @param string $identifier Unique identifier of the job. + * + * @return BatchJob|null + */ + public function getJobFromId($identifier) + { + return $this->config->getJobFromId($identifier); + } + + /** + * Get the job with the given numeric id. + * + * @param int $idNum A numeric id of the job. + * + * @return BatchJob|null + */ + public function getJobFromIdNum($idNum) + { + return $this->config->getJobFromIdNum($idNum); + } + + /** + * Get all the jobs. + * + * @return BatchJob[] + */ + public function getJobs() + { + return $this->config->getJobs(); + } + + /** + * Load the config from the storage. + * + * @return bool true on success + * @throws \RuntimeException when it fails to load the config. + */ + public function loadConfig() + { + $result = $this->configStorage->lock(); + if ($result === false) { + throw new \RuntimeException('Failed to lock the configStorage'); + } + $result = $this->configStorage->load(); + $this->configStorage->unlock(); + if ($result === false) { + throw new \RuntimeException('Failed to load the BatchConfig'); + } + $this->config = $result; + return true; + } + + /** + * Gets the item processor. + * + * @return ProcessItemInterface + */ + public function getProcessor() + { + return $this->processor; + } +} diff --git a/src/Core/Batch/BatchTrait.php b/src/Core/Batch/BatchTrait.php new file mode 100644 index 000000000000..5df9806d6f91 --- /dev/null +++ b/src/Core/Batch/BatchTrait.php @@ -0,0 +1,195 @@ +batchRunner + ->getJobFromId($this->identifier) + ->getIdNum(); + + return $this->batchRunner + ->getProcessor() + ->flush($id); + } + + /** + * Deliver a list of items in a batch call. + * + * @param array $items + * @return bool + * @access private + */ + public function send(array $items) + { + $start = microtime(true); + try { + call_user_func_array($this->getCallback(), [$items]); + } catch (\Exception $e) { + fwrite( + $this->debugOutputResource ?: STDERR, + $e->getMessage() . PHP_EOL + ); + + return false; + } + $end = microtime(true); + if ($this->debugOutput) { + fwrite( + $this->debugOutputResource ?: STDERR, + sprintf( + '%f seconds for %s: %d items' . PHP_EOL, + $end - $start, + $this->batchMethod, + count($items) + ) + ); + fwrite( + $this->debugOutputResource ?: STDERR, + sprintf( + 'memory used: %d' . PHP_EOL, + memory_get_usage() + ) + ); + } + + return true; + } + + /** + * Returns an array representation of a callback which will be used to write + * batch items. + * + * @return array + */ + protected abstract function getCallback(); + + /** + * @param array $options [optional] { + * Configuration options. + * + * @type resource $debugOutputResource A resource to output debug output + * to. + * @type bool $debugOutput Whether or not to output debug information. + * **Defaults to** false. + * @type array $batchOptions A set of options for a BatchJob. + * {@see \Google\Cloud\Core\Batch\BatchJob::__construct()} for + * more details. + * **Defaults to** ['batchSize' => 1000, + * 'callPeriod' => 2.0, + * 'workerNum' => 2]. + * @type array $clientConfig A config used to construct the client upon + * which requests will be made. + * @type BatchRunner $batchRunner A BatchRunner object. Mainly used for + * the tests to inject a mock. **Defaults to** a newly created + * BatchRunner. + * @type string $identifier An identifier for the batch job. + * @type string $batchMethod The name of the batch method used to + * deliver items. + * } + * @throws \InvalidArgumentException + */ + private function setCommonBatchProperties(array $options) + { + if (!isset($options['identifier'])) { + throw new \InvalidArgumentException( + 'A valid identifier is required in order to register a job.' + ); + } + + if (!isset($options['batchMethod'])) { + throw new \InvalidArgumentException( + 'A batchMethod is required.' + ); + } + + $this->batchMethod = $options['batchMethod']; + $this->identifier = $options['identifier']; + $this->debugOutputResource = isset($options['debugOutputResource']) + ? $options['debugOutputResource'] + : null; + $this->debugOutput = isset($options['debugOutput']) + ? $options['debugOutput'] + : false; + $this->clientConfig = isset($options['clientConfig']) + ? $options['clientConfig'] + : []; + $batchOptions = isset($options['batchOptions']) + ? $options['batchOptions'] + : []; + $this->batchOptions = $batchOptions + [ + 'batchSize' => 1000, + 'callPeriod' => 2.0, + 'workerNum' => 2 + ]; + $this->batchRunner = isset($options['batchRunner']) + ? $options['batchRunner'] + : new BatchRunner(); + $this->batchRunner->registerJob( + $this->identifier, + [$this, 'send'], + $this->batchOptions + ); + } +} diff --git a/src/Core/Batch/ConfigStorageInterface.php b/src/Core/Batch/ConfigStorageInterface.php new file mode 100644 index 000000000000..cff9bbbb6bd1 --- /dev/null +++ b/src/Core/Batch/ConfigStorageInterface.php @@ -0,0 +1,53 @@ +baseDir = getenv('GOOGLE_CLOUD_BATCH_DAEMON_FAILURE_DIR'); + if ($this->baseDir === false) { + $this->baseDir = sprintf( + '%s/batch-daemon-failure', + sys_get_temp_dir() + ); + } + if (! is_dir($this->baseDir)) { + if (@mkdir($this->baseDir, 0700, true) === false) { + throw new \RuntimeException( + sprintf( + 'Couuld not create a directory: %s', + $this->baseDir + ) + ); + } + } + // Use getmypid for simplicity. + $this->failureFile = sprintf( + '%s/failed-items-%d', + $this->baseDir, + getmypid() + ); + } + + /** + * Save the items to the failureFile. We silently abandon the items upon + * failures in this method because there's nothing we can do. + * + * @param int $idNum A numeric id for the job. + * @param array $items Items to save. + */ + public function handleFailure($idNum, array $items) + { + $fp = @fopen($this->failureFile, 'a'); + @fwrite($fp, serialize([$idNum => $items]) . PHP_EOL); + @fclose($fp); + } + + /** + * Get all the filenames for the failure files. + * + * @return array Filenames for all the failure files. + */ + private function getFailedFiles() + { + $pattern = sprintf('%s/failed-items-*', $this->baseDir); + return glob($pattern) ?: []; + } +} diff --git a/src/Core/Batch/InMemoryConfigStorage.php b/src/Core/Batch/InMemoryConfigStorage.php new file mode 100644 index 000000000000..95b249a0f783 --- /dev/null +++ b/src/Core/Batch/InMemoryConfigStorage.php @@ -0,0 +1,203 @@ +config = new BatchConfig(); + $this->created = microtime(true); + $this->initFailureFile(); + $this->hasShutdownHookRegistered = false; + } + + /** + * Just return true + * + * @return bool + */ + public function lock() + { + return true; + } + + /** + * Just return true + * + * @return bool + */ + public function unlock() + { + return true; + } + + /** + * Save the given BatchConfig. + * + * @param BatchConfig $config A BatchConfig to save. + * @return bool + */ + public function save(BatchConfig $config) + { + $this->config = $config; + return true; + } + + /** + * Load a BatchConfig from the storage. + * + * @return BatchConfig + * @throws \RuntimeException when failed to load the BatchConfig. + */ + public function load() + { + return $this->config; + } + + /** + * Hold the items in memory and run the job in the same process when it + * meets the condition. + * + * We want to delay registering the shutdown function. The error + * reporter also registers a shutdown function and the order matters. + * {@see Google\ErrorReporting\Bootstrap::init()} + * {@see http://php.net/manual/en/function.register-shutdown-function.php} + * + * @param mixed $item An item to submit. + * @param int $idNum A numeric id for the job. + * @return void + */ + public function submit($item, $idNum) + { + if (!$this->hasShutdownHookRegistered) { + register_shutdown_function([$this, 'shutdown']); + $this->hasShutdownHookRegistered = true; + } + if (!array_key_exists($idNum, $this->items)) { + $this->items[$idNum] = []; + $this->lastInvoked[$idNum] = $this->created; + } + $this->items[$idNum][] = $item; + $job = $this->config->getJobFromIdNum($idNum); + $batchSize = $job->getBatchSize(); + $period = $job->getCallPeriod(); + if ((count($this->items[$idNum]) >= $batchSize) + || (count($this->items[$idNum]) !== 0 + && microtime(true) > $this->lastInvoked[$idNum] + $period)) { + $this->flush($idNum); + $this->items[$idNum] = []; + $this->lastInvoked[$idNum] = microtime(true); + } + } + + /** + * Run the job with the given id. + * + * @param int $idNum A numeric id for the job. + * @return bool + */ + public function flush($idNum) + { + if (isset($this->items[$idNum])) { + $job = $this->config->getJobFromIdNum($idNum); + + if (!$job->run($this->items[$idNum])) { + $this->handleFailure($idNum, $this->items[$idNum]); + } + + $this->items[$idNum] = []; + $this->lastInvoked[$idNum] = microtime(true); + } + + return true; + } + + /** + * Run the job for remainder items. + */ + public function shutdown() + { + foreach ($this->items as $idNum => $items) { + if (count($items) !== 0) { + $this->flush($idNum); + } + } + } +} diff --git a/src/Core/Batch/ProcessItemInterface.php b/src/Core/Batch/ProcessItemInterface.php new file mode 100644 index 000000000000..76d6919e63bd --- /dev/null +++ b/src/Core/Batch/ProcessItemInterface.php @@ -0,0 +1,42 @@ +runner = $runner ?: new BatchRunner(); + $this->initFailureFile(); + } + + /** + * Retry all the failed items. + */ + public function retryAll() + { + foreach ($this->getFailedFiles() as $file) { + // Rename the file first + $tmpFile = dirname($file) . '/retrying-' . basename($file); + rename($file, $tmpFile); + + $fp = @fopen($tmpFile, 'r'); + if ($fp === false) { + fwrite( + STDERR, + sprintf('Could not open the file: %s' . PHP_EOL, $tmpFile) + ); + continue; + } + while ($line = fgets($fp)) { + $a = unserialize($line); + $idNum = key($a); + $job = $this->runner->getJobFromIdNum($idNum); + if (! $job->run($a[$idNum])) { + $this->handleFailure($idNum, $a[$idNum]); + } + } + @fclose($fp); + @unlink($tmpFile); + } + } +} diff --git a/src/Core/Batch/SysvConfigStorage.php b/src/Core/Batch/SysvConfigStorage.php new file mode 100644 index 000000000000..2cfc72b4c68a --- /dev/null +++ b/src/Core/Batch/SysvConfigStorage.php @@ -0,0 +1,103 @@ +sysvKey = ftok(__FILE__, 'A'); + $this->semid = sem_get($this->sysvKey, 1, 0600, 1); + } + + /** + * Acquire a lock. + * + * @return bool + */ + public function lock() + { + return sem_acquire($this->semid); + } + + /** + * Release a lock. + * + * @return bool + */ + public function unlock() + { + return sem_release($this->semid); + } + + /** + * Save the given BatchConfig. + * + * @param BatchConfig $config A BatchConfig to save. + * @return bool + * @throws \RuntimeException when failed to attach to the shared memory. + */ + public function save(BatchConfig $config) + { + $shmid = shm_attach($this->sysvKey); + if ($shmid === false) { + throw new \RuntimeException( + 'Failed to attach to the shared memory' + ); + } + $result = shm_put_var($shmid, self::VAR_KEY, $config); + shm_detach($shmid); + return $result; + } + + /** + * Load a BatchConfig from the storage. + * + * @return BatchConfig + * @throws \RuntimeException when failed to attach to the shared memory. + */ + public function load() + { + $shmid = shm_attach($this->sysvKey); + if ($shmid === false) { + throw new \RuntimeException( + 'Failed to attach to the shared memory' + ); + } + if (! shm_has_var($shmid, self::VAR_KEY)) { + $result = new BatchConfig(); + } else { + $result = shm_get_var($shmid, self::VAR_KEY); + } + return $result; + } +} diff --git a/src/Core/Batch/SysvProcessor.php b/src/Core/Batch/SysvProcessor.php new file mode 100644 index 000000000000..03886f89a85a --- /dev/null +++ b/src/Core/Batch/SysvProcessor.php @@ -0,0 +1,84 @@ +sysvQs)) { + $this->sysvQs[$idNum] = + msg_get_queue($this->getSysvKey($idNum)); + } + $result = @msg_send( + $this->sysvQs[$idNum], + self::$typeDirect, + $item + ); + if ($result === false) { + // Try to put the content in a temp file and send the filename. + $tempFile = tempnam(sys_get_temp_dir(), 'Item'); + $result = file_put_contents($tempFile, serialize($item)); + if ($result === false) { + throw new \RuntimeException( + "Failed to write to $tempFile while submiting the item" + ); + } + $result = @msg_send( + $this->sysvQs[$idNum], + self::$typeFile, + $tempFile + ); + if ($result === false) { + @unlink($tempFile); + throw new \RuntimeException( + "Failed to submit the filename: $tempFile" + ); + } + } + } + + /** + * Run the job with the given id. This has no effect and simply always + * returns false when using the batch daemon. + * + * @param int $idNum A numeric id of the job. + * @return bool + */ + public function flush($idNum) + { + return false; + } +} diff --git a/src/Core/Batch/SysvTrait.php b/src/Core/Batch/SysvTrait.php new file mode 100644 index 000000000000..943065223cf7 --- /dev/null +++ b/src/Core/Batch/SysvTrait.php @@ -0,0 +1,74 @@ +onGce($config['httpHandler'])) { $metadata = $this->getMetaData(); $projectId = $metadata->getProjectId(); diff --git a/src/Core/Report/EmptyMetadataProvider.php b/src/Core/Report/EmptyMetadataProvider.php new file mode 100644 index 000000000000..50c07e92e853 --- /dev/null +++ b/src/Core/Report/EmptyMetadataProvider.php @@ -0,0 +1,72 @@ + + substr($server['HTTP_X_CLOUD_TRACE_CONTEXT'], 0, 32)] + : []; + $this->data = + [ + 'resource' => [ + 'type' => 'gae_app', + 'labels' => [ + 'project_id' => $projectId, + 'version_id' => $versionId, + 'module_id' => $serviceId + ] + ], + 'projectId' => $projectId, + 'serviceId' => $serviceId, + 'versionId' => $versionId, + 'labels' => $labels + ]; + } + + /** + * Return an array representing MonitoredResource. + * {@see https://cloud.google.com/logging/docs/reference/v2/rest/v2/MonitoredResource} + * + * @return array + */ + public function monitoredResource() + { + return $this->data['resource']; + } + + /** + * Return the project id. + * @return string + */ + public function projectId() + { + return $this->data['projectId']; + } + + /** + * Return the service id. + * @return string + */ + public function serviceId() + { + return $this->data['serviceId']; + } + + /** + * Return the version id. + * @return string + */ + public function versionId() + { + return $this->data['versionId']; + } + + /** + * Return the labels. We need to evaluate $_SERVER for each request. + * @return array + */ + public function labels() + { + return $this->data['labels']; + } +} diff --git a/src/Core/Report/MetadataProviderInterface.php b/src/Core/Report/MetadataProviderInterface.php new file mode 100644 index 000000000000..9834fda0ddbf --- /dev/null +++ b/src/Core/Report/MetadataProviderInterface.php @@ -0,0 +1,56 @@ +data['monitoredResource'] = $monitoredResource; + $this->data['projectId'] = $projectId; + $this->data['serviceId'] = $serviceId; + $this->data['versionId'] = $versionId; + $this->data['labels'] = $labels; + } + + /** + * Return an array representing MonitoredResource. + * {@see https://cloud.google.com/logging/docs/reference/v2/rest/v2/MonitoredResource} + * + * @return array + */ + public function monitoredResource() + { + return $this->data['monitoredResource']; + } + + /** + * Return the project id. + * @return string + */ + public function projectId() + { + return $this->data['projectId']; + } + + /** + * Return the service id. + * @return string + */ + public function serviceId() + { + return $this->data['serviceId']; + } + + /** + * Return the version id. + * @return string + */ + public function versionId() + { + return $this->data['versionId']; + } + + /** + * Return the labels. + * @return array + */ + public function labels() + { + return $this->data['labels']; + } +} diff --git a/src/Core/bin/google-cloud-batch b/src/Core/bin/google-cloud-batch new file mode 100755 index 000000000000..7e0a161f4937 --- /dev/null +++ b/src/Core/bin/google-cloud-batch @@ -0,0 +1,57 @@ +#!/usr/bin/env php +runParent(); + } else { + $idNum = (int) $argv[2]; + $daemon->runChild($idNum); + } +} elseif ($argv[1] === 'retry') { + $retry = new Retry(); + $retry->retryAll(); +} else { + showUsageAndDie(); +} diff --git a/src/Core/composer.json b/src/Core/composer.json index d05ecb4f0908..fa0ab039a59c 100644 --- a/src/Core/composer.json +++ b/src/Core/composer.json @@ -23,6 +23,9 @@ "entry": null } }, + "bin": [ + "bin/google-cloud-batch" + ], "autoload": { "psr-4": { "Google\\Cloud\\Core\\": "" diff --git a/src/ErrorReporting/Bootstrap.php b/src/ErrorReporting/Bootstrap.php new file mode 100644 index 000000000000..e7e0e22bd044 --- /dev/null +++ b/src/ErrorReporting/Bootstrap.php @@ -0,0 +1,218 @@ +psrLogger(self::DEFAULT_LOGNAME, [ + 'batchEnabled' => true, + 'debugOutput' => true, + 'batchOptions' => [ + 'workerNum' => 2 + ] + ]); + register_shutdown_function([self::class, 'shutdownHandler']); + set_exception_handler([self::class, 'exceptionHandler']); + set_error_handler([self::class, 'errorHandler']); + } + + /** + * Return a string prefix for the given error level. + * + * @param int $level + * @return string A string prefix for reporting the error. + */ + public static function getErrorPrefix($level) + { + switch ($level) { + case E_PARSE: + $prefix = 'PHP Parse error'; + break; + case E_ERROR: + case E_CORE_ERROR: + case E_COMPILE_ERROR: + $prefix = 'PHP Fatal error'; + break; + case E_USER_ERROR: + case E_RECOVERABLE_ERROR: + $prefix = 'PHP error'; + break; + case E_WARNING: + case E_CORE_WARNING: + case E_COMPILE_WARNING: + case E_USER_WARNING: + $prefix = 'PHP Warning'; + break; + case E_NOTICE: + case E_USER_NOTICE: + $prefix = 'PHP Notice'; + break; + case E_STRICT: + $prefix = 'PHP Debug'; + break; + default: + $prefix = 'PHP Notice'; + } + return $prefix; + } + + /** + * Return an error level string for the given PHP error level. + * + * @param int $level + * @return string An error level string. + */ + public static function getErrorLevelString($level) + { + switch ($level) { + case E_PARSE: + return 'CRITICAL'; + case E_ERROR: + case E_CORE_ERROR: + case E_COMPILE_ERROR: + case E_USER_ERROR: + case E_RECOVERABLE_ERROR: + return 'ERROR'; + case E_WARNING: + case E_CORE_WARNING: + case E_COMPILE_WARNING: + case E_USER_WARNING: + return 'WARNING'; + case E_NOTICE: + case E_USER_NOTICE: + return 'NOTICE'; + case E_STRICT: + return 'DEBUG'; + default: + return 'NOTICE'; + } + return $prefix; + } + + /** + * @param mixed $ex \Throwable (PHP 7) or \Exception (PHP 5) + */ + public static function exceptionHandler($ex) + { + $message = sprintf('PHP Notice: %s', (string)$ex); + if (self::$psrBatchLogger) { + self::$psrBatchLogger->error($message); + } else { + fwrite(STDERR, $message . PHP_EOL); + } + } + + /** + * @param int $level The error level. + * @param string $message The error message. + * @param string $file The filename that the error was raised in. + * @param int $line The line number that the error was raised at. + */ + public static function errorHandler($level, $message, $file, $line) + { + if (!($level & error_reporting())) { + return true; + } + $message = sprintf( + '%s: %s in %s on line %d', + self::getErrorPrefix($level), + $message, + $file, + $line + ); + if (!self::$psrBatchLogger) { + return false; + } + $service = self::$psrBatchLogger->getMetadataProvider()->serviceId(); + $version = self::$psrBatchLogger->getMetadataProvider()->versionId(); + $context = [ + 'context' => [ + 'reportLocation' => [ + 'filePath' => $file, + 'lineNumber' => $line, + 'functionName' => 'unknown' + ] + ], + 'serviceContext' => [ + 'service' => $service, + 'version' => $version + ] + ]; + self::$psrBatchLogger->log( + self::getErrorLevelString($level), + $message, + $context + ); + } + + /** + * Called at exit, to check there's a fatal error and report the error if + * any. + */ + public static function shutdownHandler() + { + if ($err = error_get_last()) { + switch ($err['type']) { + case E_ERROR: + case E_PARSE: + case E_COMPILE_ERROR: + case E_CORE_ERROR: + $service = self::$psrBatchLogger + ->getMetadataProvider() + ->serviceId(); + $version = self::$psrBatchLogger + ->getMetadataProvider() + ->versionId(); + $message = sprintf( + '%s: %s in %s on line %d', + self::getErrorPrefix($err['type']), + $err['message'], + $err['file'], + $err['line'] + ); + $context = [ + 'context' => [ + 'reportLocation' => [ + 'filePath' => $err['file'], + 'lineNumber' => $err['line'], + 'functionName' => 'unknown' + ] + ], + 'serviceContext' => [ + 'service' => $service, + 'version' => $version + ] + ]; + if (self::$psrBatchLogger) { + self::$psrBatchLogger->log( + self::getErrorLevelString($err['type']), + $message, + $context + ); + } + break; + } + } + } +} diff --git a/src/ErrorReporting/prepend.php b/src/ErrorReporting/prepend.php new file mode 100644 index 000000000000..195f0cc14c25 --- /dev/null +++ b/src/ErrorReporting/prepend.php @@ -0,0 +1,68 @@ +connection = $connection; + $this->name = $name; $this->formattedName = "projects/$projectId/logs/$name"; $this->projectId = $projectId; $this->resource = $resource ?: ['type' => 'global']; @@ -383,6 +389,16 @@ public function writeBatch(array $entries, array $options = []) $this->connection->writeEntries($options + ['entries' => $entries]); } + /** + * Returns the logger's name. + * + * @return string + */ + public function name() + { + return $this->name; + } + /** * Returns the log level map. * diff --git a/src/Logging/LoggingClient.php b/src/Logging/LoggingClient.php index d5a03d854199..0baf663c6fd8 100644 --- a/src/Logging/LoggingClient.php +++ b/src/Logging/LoggingClient.php @@ -21,6 +21,7 @@ use Google\Cloud\Core\ClientTrait; use Google\Cloud\Core\Iterator\ItemIterator; use Google\Cloud\Core\Iterator\PageIterator; +use Google\Cloud\Core\Report\MetadataProviderInterface; use Google\Cloud\Logging\Connection\ConnectionInterface; use Google\Cloud\Logging\Connection\Grpc; use Google\Cloud\Logging\Connection\Rest; @@ -83,6 +84,56 @@ class LoggingClient */ private $formattedProjectName; + /** + * @var array The config given to the constructor. + */ + private $config; + + /** + * Create a PsrLogger with batching enabled. + * + * @param string $name The name of the log to write entries to. + * @param array $options [optional] { + * Configuration options. + * + * @type string $messageKey The key in the `jsonPayload` used to contain + * the logged message. **Defaults to** `message`. + * @type array $resource The + * [monitored resource](https://cloud.google.com/logging/docs/api/reference/rest/v2/MonitoredResource) + * to associate log entries with. **Defaults to** type global. + * @type array $labels A set of user-defined (key, value) data that + * provides additional information about the log entry. + * @type MetadataProviderInterface $metadataProvider **Defaults to** An + * automatically chosen provider, based on detected environment + * settings. + * @type bool $debugOutput Whether or not to output debug information. + * **Defaults to** false. + * @type array $batchOptions A set of options for a BatchJob. + * {@see \Google\Cloud\Core\Batch\BatchJob::__construct()} for + * more details. + * **Defaults to** ['batchSize' => 1000, + * 'callPeriod' => 2.0, + * 'workerNum' => 2]. + * @type array $clientConfig Configuration options for the Logging client + * used to handle processing of batch items. For valid options + * please see + * {@see \Google\Cloud\Logging\LoggingClient::__construct()}. + * @type BatchRunner $batchRunner A BatchRunner object. Mainly used for + * the tests to inject a mock. **Defaults to** a newly created + * BatchRunner. + * } + * @return PsrLogger + **/ + public static function psrBatchLogger($name, array $options = []) + { + $client = array_key_exists('clientConfig', $options) + ? new self($options['clientConfig']) + : new self(); + // Force enabling batch. + $options['batchEnabled'] = true; + return $client->psrLogger($name, $options); + } + /** * Create a Logging client. * @@ -113,6 +164,7 @@ class LoggingClient */ public function __construct(array $config = []) { + $this->config = $config; $connectionType = $this->getConnectionType($config); if (!isset($config['scopes'])) { $config['scopes'] = [self::FULL_CONTROL_SCOPE]; @@ -422,6 +474,13 @@ function (array $entry) { * $psrLogger = $logging->psrLogger('my-log'); * ``` * + * ``` + * // Write entries with background batching. + * $psrLogger = $logging->psrLogger('my-log', [ + * 'batchEnabled' => true + * ]); + * ``` + * * @param string $name The name of the log to write entries to. * @param array $options [optional] { * Configuration options. @@ -433,6 +492,30 @@ function (array $entry) { * to associate log entries with. **Defaults to** type global. * @type array $labels A set of user-defined (key, value) data that * provides additional information about the log entry. + * @type MetadataProviderInterface $metadataProvider **Defaults to** An + * automatically chosen provider, based on detected environment + * settings. + * @type bool $batchEnabled Determines whether or not to use background + * batching. **Defaults to** `false`. + * @type bool $debugOutput Whether or not to output debug information. + * **Defaults to** false. Applies only when `batchEnabled` is set + * to `true`. + * @type array $batchOptions A set of options for a BatchJob. + * {@see \Google\Cloud\Core\Batch\BatchJob::__construct()} for + * more details. + * **Defaults to** ['batchSize' => 1000, + * 'callPeriod' => 2.0, + * 'workerNum' => 2]. Applies only when + * `batchEnabled` is set to `true`. + * @type array $clientConfig Configuration options for the Logging client + * used to handle processing of batch items. For valid options + * please see + * {@see \Google\Cloud\Logging\LoggingClient::__construct()}. + * **Defaults to** the options provided to the current client. + * Applies only when `batchEnabled` is set to `true`. + * @type BatchRunner $batchRunner A BatchRunner object. Mainly used for + * the tests to inject a mock. **Defaults to** a newly created + * BatchRunner. Applies only when `batchEnabled` is set to `true`. * } * @return PsrLogger */ @@ -445,9 +528,22 @@ public function psrLogger($name, array $options = []) unset($options['messageKey']); } - return $messageKey - ? new PsrLogger($this->logger($name, $options), $messageKey) - : new PsrLogger($this->logger($name, $options)); + $psrLoggerOptions = $this->pluckArray([ + 'metadataProvider', + 'batchEnabled', + 'debugOutput', + 'batchOptions', + 'clientConfig', + 'batchRunner' + ], $options); + + return new PsrLogger( + $this->logger($name, $options), + $messageKey, + $psrLoggerOptions + [ + 'clientConfig' => $this->config + ] + ); } /** diff --git a/src/Logging/PsrLogger.php b/src/Logging/PsrLogger.php index d39f90ef0ea7..3de8ff0b589b 100644 --- a/src/Logging/PsrLogger.php +++ b/src/Logging/PsrLogger.php @@ -17,11 +17,13 @@ namespace Google\Cloud\Logging; +use Google\Cloud\Core\Batch\BatchTrait; +use Google\Cloud\Core\Report\MetadataProviderInterface; +use Google\Cloud\Core\Report\MetadataProviderUtils; use Monolog\Formatter\NormalizerFormatter; use Monolog\Processor\PsrLogMessageProcessor; use Psr\Log\InvalidArgumentException; use Psr\Log\LoggerInterface; -use Psr\Log\LogLevel; /** * A PSR-3 compliant logger used to write entries to Google Stackdriver Logging. @@ -35,10 +37,30 @@ * $psrLogger = $logging->psrLogger('my-log'); * ``` * + * ``` + * // Write entries with background batching. + * use Google\Cloud\Logging\LoggingClient; + * + * $logging = new LoggingClient(); + * + * $psrLogger = $logging->psrLogger('my-log', [ + * 'batchEnabled' => true + * ]); + * ``` + * * @see http://www.php-fig.org/psr/psr-3/#psrlogloggerinterface Psr\Log\LoggerInterface */ -class PsrLogger implements LoggerInterface +class PsrLogger implements LoggerInterface, \Serializable { + use BatchTrait; + + const ID_TEMPLATE = 'stackdriver-logging-%s'; + + /** + * @var array + */ + private static $loggers = []; + /** * @var Logger The logger used to write entries. */ @@ -49,15 +71,67 @@ class PsrLogger implements LoggerInterface */ private $messageKey; + /** + * @var bool + */ + private $batchEnabled; + + /** + * @var MetadataProviderInterface + */ + private $metadataProvider; + /** * @param Logger $logger The logger used to write entries. * @param string $messageKey The key in the `jsonPayload` used to contain * the logged message. **Defaults to** `message`. + * @param array $options [optional] { + * Configuration options. + * + * @type MetadataProviderInterface $metadataProvider **Defaults to** An + * automatically chosen provider, based on detected environment + * settings. + * @type bool $batchEnabled Determines whether or not to use background + * batching. **Defaults to** `false`. + * @type bool $debugOutput Whether or not to output debug information. + * **Defaults to** false. Applies only when `batchEnabled` is set + * to `true`. + * @type array $batchOptions A set of options for a BatchJob. + * {@see \Google\Cloud\Core\Batch\BatchJob::__construct()} for + * more details. + * **Defaults to** ['batchSize' => 1000, + * 'callPeriod' => 2.0, + * 'workerNum' => 2]. Applies only when + * `batchEnabled` is set to `true`. + * @type array $clientConfig Configuration options for the Logging client + * used to handle processing of batch items. For valid options + * please see + * {@see \Google\Cloud\Logging\LoggingClient::__construct()}. + * Applies only when `batchEnabled` is set to `true`. + * @type BatchRunner $batchRunner A BatchRunner object. Mainly used for + * the tests to inject a mock. **Defaults to** a newly created + * BatchRunner. Applies only when `batchEnabled` is set to `true`. + * } */ - public function __construct(Logger $logger, $messageKey = 'message') - { + public function __construct( + Logger $logger, + $messageKey = null, + array $options = [] + ) { $this->logger = $logger; - $this->messageKey = $messageKey; + $this->logName = $logger->name(); + $this->messageKey = $messageKey ?: 'message'; + $this->metadataProvider = isset($options['metadataProvider']) + ? $options['metadataProvider'] + : MetadataProviderUtils::autoSelect($_SERVER); + + if (isset($options['batchEnabled']) && $options['batchEnabled'] === true) { + $this->batchEnabled = true; + $this->setCommonBatchProperties($options + [ + 'identifier' => sprintf(self::ID_TEMPLATE, $this->logger->name()), + 'batchMethod' => 'writeBatch' + ]); + } } /** @@ -289,14 +363,92 @@ public function log($level, $message, array $context = []) ]); $jsonPayload = [$this->messageKey => $processedData['message']]; + // Adding labels for log request correlation. + $labels = $this->getLabels(); + if (! empty($labels)) { + $options['labels'] = + (isset($options['labels']) + ? $options['labels'] + : []) + $labels; + } + // Adding MonitoredResource + $resource = $this->metadataProvider->monitoredResource(); + if (! empty($resource)) { + $options['resource'] = + (isset($options['resource']) + ? $options['resource'] + : []) + $resource; + } $entry = $this->logger->entry( $jsonPayload + $processedData['context'], $options + [ 'severity' => $level ] ); + $this->sendEntry($entry); + } - $this->logger->write($entry); + /** + * Return the MetadataProvider. + * + * @return MetadataProviderInterface + */ + public function getMetadataProvider() + { + return $this->metadataProvider; + } + + /** + * Serializes data. + * + * @return string + * @access private + */ + public function serialize() + { + return serialize([ + $this->messageKey, + $this->batchEnabled, + $this->metadataProvider, + $this->debugOutput, + $this->clientConfig, + $this->batchMethod, + $this->logName + ]); + } + + /** + * Unserializes data. + * + * @param string + * @access private + */ + public function unserialize($data) + { + list( + $this->messageKey, + $this->batchEnabled, + $this->metadataProvider, + $this->debugOutput, + $this->clientConfig, + $this->batchMethod, + $this->logName + ) = unserialize($data); + } + + /** + * Returns an array representation of a callback which will be used to write + * batch items. + * + * @return array + */ + protected function getCallback() + { + if (!array_key_exists($this->logName, self::$loggers)) { + $c = new LoggingClient($this->clientConfig); + self::$loggers[$this->logName] = $c->logger($this->logName); + } + return [self::$loggers[$this->logName], $this->batchMethod]; } /** @@ -308,7 +460,7 @@ public function log($level, $message, array $context = []) */ private function validateLogLevel($level) { - $map = $this->logger->getLogLevelMap(); + $map = Logger::getLogLevelMap(); $level = (string) $level; if (isset($map[$level]) || isset(array_flip($map)[strtoupper($level)])) { @@ -317,4 +469,30 @@ private function validateLogLevel($level) throw new InvalidArgumentException("Severity level '$level' is not defined."); } + + /** + * Send the given entry. + * + * @param Entry $entry + */ + private function sendEntry(Entry $entry) + { + if ($this->batchEnabled) { + $this->batchRunner->submitItem($this->identifier, $entry); + return; + } + + $this->logger->write($entry); + } + + /** + * Return additional labels. Now it returns labels for log request + * correlation. + * + * @return array + */ + private function getLabels() + { + return $this->metadataProvider->labels(); + } } diff --git a/src/PubSub/BatchPublisher.php b/src/PubSub/BatchPublisher.php new file mode 100644 index 000000000000..2b3657c56b3d --- /dev/null +++ b/src/PubSub/BatchPublisher.php @@ -0,0 +1,98 @@ +topic('my_topic') + * ->batchPublisher(); + * + * $batchPublisher->publish([ + * 'data' => 'An important message.' + * ]); + * ``` + */ +class BatchPublisher +{ + use BatchTrait; + + const ID_TEMPLATE = 'pubsub-topic-%s'; + + /** + * @var array + */ + private static $topics = []; + + /** + * @param string $topicName The topic name. + * @param array $options [optional] Please see + * {@see Google\Cloud\PubSub\Topic::batchPublisher()} for + * configuration details. + */ + public function __construct($topicName, array $options = []) + { + $this->topicName = $topicName; + $this->setCommonBatchProperties($options + [ + 'identifier' => sprintf(self::ID_TEMPLATE, $topicName), + 'batchMethod' => 'publishBatch' + ]); + } + + /** + * Send messages to a batch queue. + * + * Example: + * ``` + * $batchPublisher->publish([ + * 'data' => 'An important message.' + * ]); + * ``` + * + * @param array $message [Message Format](https://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage). + * @return bool + */ + public function publish(array $message) + { + return $this->batchRunner->submitItem($this->identifier, $message); + } + + /** + * Returns an array representation of a callback which will be used to write + * batch items. + * + * @return array + */ + protected function getCallback() + { + if (!array_key_exists($this->topicName, self::$topics)) { + $client = new PubSubClient($this->clientConfig); + self::$topics[$this->topicName] = $client->topic($this->topicName); + } + + return [self::$topics[$this->topicName], $this->batchMethod]; + } +} diff --git a/src/PubSub/PubSubClient.php b/src/PubSub/PubSubClient.php index c9d00cab2eae..da46495ead23 100644 --- a/src/PubSub/PubSubClient.php +++ b/src/PubSub/PubSubClient.php @@ -99,6 +99,11 @@ class PubSubClient */ private $encode; + /** + * @var array + */ + private $clientConfig; + /** * Create a PubSub client. * @@ -130,6 +135,7 @@ class PubSubClient */ public function __construct(array $config = []) { + $this->clientConfig = $config; $connectionType = $this->getConnectionType($config); if (!isset($config['scopes'])) { $config['scopes'] = [self::FULL_CONTROL_SCOPE]; @@ -512,7 +518,8 @@ private function topicFactory($name, array $info = []) $this->projectId, $name, $this->encode, - $info + $info, + $this->clientConfig ); } @@ -523,7 +530,7 @@ private function topicFactory($name, array $info = []) * @param string $name The subscription name * @param string $topicName [optional] The topic name * @param array $info [optional] Information about the subscription. Used - * to populate subscriptons with an api result. Should be a + * to populate subscriptons with an API result. Should be a * representation of a [Subscription](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions#Subscription). * @return Subscription * @codingStandardsIgnoreEnd diff --git a/src/PubSub/Snapshot.php b/src/PubSub/Snapshot.php index 91f01d9d57f3..50e9eb268346 100644 --- a/src/PubSub/Snapshot.php +++ b/src/PubSub/Snapshot.php @@ -61,6 +61,11 @@ class Snapshot */ private $info; + /** + * @var array + */ + private $clientConfig; + /** * @param ConnectionInterface $connection A connection to Cloud Pub/Sub * @param string $projectId The current Project ID. @@ -69,12 +74,25 @@ class Snapshot * @param array $info [optional] The snapshot data. When creating a * Snapshot, this array **must** contain a `$info.subscription` * argument with a fully-qualified subscription name. + * @param array $clientConfig [optional] Configuration options for the + * PubSub client used to handle processing of batch items through the + * daemon. For valid options please see + * {@see \Google\Cloud\PubSub\PubSubClient::__construct()}. + * **Defaults to** the options provided to the PubSub client + * associated with this instance. */ - public function __construct(ConnectionInterface $connection, $projectId, $name, $encode, array $info = []) - { + public function __construct( + ConnectionInterface $connection, + $projectId, + $name, + $encode, + array $info = [], + array $clientConfig = [] + ) { $this->connection = $connection; $this->projectId = $projectId; $this->encode = $encode; + $this->clientConfig = $clientConfig; // Accept either a simple name or a fully-qualified name. if ($this->isFullyQualifiedName('snapshot', $name)) { @@ -183,9 +201,18 @@ public function delete(array $options = []) */ public function topic() { - return $this->info['topic'] - ? new Topic($this->connection, $this->projectId, $this->info['topic'], $this->encode) - : null; + if ($this->info['topic']) { + return new Topic( + $this->connection, + $this->projectId, + $this->info['topic'], + $this->encode, + [], + $this->clientConfig + ); + } + + return null; } /** diff --git a/src/PubSub/Topic.php b/src/PubSub/Topic.php index e026d8d25d31..29612a1ec156 100644 --- a/src/PubSub/Topic.php +++ b/src/PubSub/Topic.php @@ -77,6 +77,11 @@ class Topic */ private $iam; + /** + * @var array + */ + private $clientConfig; + /** * Create a PubSub topic. * @@ -86,18 +91,26 @@ class Topic * @param string $name The topic name * @param bool $encode Whether messages should be base64 encoded. * @param array $info [optional] A [Topic](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics) + * @param array $clientConfig [optional] Configuration options for the + * PubSub client used to handle processing of batch items through the + * daemon. For valid options please see + * {@see \Google\Cloud\PubSub\PubSubClient::__construct()}. + * **Defaults to** the options provided to the PubSub client + * associated with this instance. */ public function __construct( ConnectionInterface $connection, $projectId, $name, $encode, - array $info = [] + array $info = [], + array $clientConfig = [] ) { $this->connection = $connection; $this->projectId = $projectId; $this->encode = (bool) $encode; $this->info = $info; + $this->clientConfig = $clientConfig; // Accept either a simple name or a fully-qualified name. if ($this->isFullyQualifiedName('topic', $name)) { @@ -277,7 +290,7 @@ public function reload(array $options = []) * * @see https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/publish Publish Message * - * @param array $message [Message Format](https://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage) + * @param array $message [Message Format](https://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage). * @param array $options [optional] Configuration Options * @return array A list of message IDs */ @@ -314,10 +327,6 @@ public function publish(array $message, array $options = []) * * @param array $messages A list of messages. Each message must be in the correct * [Message Format](https://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage). - * If provided, $data will be base64 encoded before being published, - * unless `$options['encode']` is set to false. (See below for more - * information.) - * } * @param array $options [optional] Configuration Options * @return array A list of message IDs. */ @@ -333,6 +342,58 @@ public function publishBatch(array $messages, array $options = []) ]); } + /** + * Push a message into a batch queue, to be processed at a later point. + * + * Example: + * ``` + * $topic->batchPublisher() + * ->publish([ + * 'data' => 'New User Registered', + * 'attributes' => [ + * 'id' => '2', + * 'userName' => 'Dave', + * 'location' => 'Detroit' + * ] + * ]); + * ``` + * + * @param array $options [optional] { + * Configuration options. + * + * @type bool $debugOutput Whether or not to output debug information. + * **Defaults to** `false`. + * @type array $batchOptions A set of options for a BatchJob. + * {@see \Google\Cloud\Core\Batch\BatchJob::__construct()} for + * more details. + * **Defaults to** ['batchSize' => 1000, + * 'callPeriod' => 2.0, + * 'workerNum' => 2]. + * @type array $clientConfig Configuration options for the PubSub client + * used to handle processing of batch items. For valid options + * please see + * {@see \Google\Cloud\PubSub\PubSubClient::__construct()}. + * **Defaults to** the options provided to the client associated + * with the current `Topic` instance. + * @type BatchRunner $batchRunner A BatchRunner object. Mainly used for + * the tests to inject a mock. **Defaults to** a newly created + * BatchRunner. + * @type string $identifier An identifier for the batch job. + * **Defaults to** `pubsub-topic-{topic-name}`. + * Example: `pubsub-topic-mytopic`. + * } + * @return BatchPublisher + */ + public function batchPublisher(array $options = []) + { + return new BatchPublisher( + $this->name, + $options + [ + 'clientConfig' => $this->clientConfig + ] + ); + } + /** * Create a subscription to the topic. * @@ -448,6 +509,7 @@ public function iam() /** * Present a nicer debug result to people using php 5.6 or greater. + * * @return array * @codeCoverageIgnore * @access private @@ -465,6 +527,7 @@ public function __debugInfo() /** * Ensure that the message is in a correct format, * base64_encode the data, and error if the input is too wrong to proceed. + * * @param array $message * @return array The message data * @throws \InvalidArgumentException diff --git a/src/Trace/README.md b/src/Trace/README.md index a033017961ac..f1156a265ecc 100644 --- a/src/Trace/README.md +++ b/src/Trace/README.md @@ -22,6 +22,7 @@ $ composer require google/cloud-trace ```php use Google\Cloud\Trace\TraceClient; use Google\Cloud\Trace\Reporter\SyncReporter; +use Google\Cloud\Trace\RequestTracer; $trace = new TraceClient(); $reporter = new SyncReporter($trace); diff --git a/src/Trace/Reporter/AsyncReporter.php b/src/Trace/Reporter/AsyncReporter.php new file mode 100644 index 000000000000..018065b05446 --- /dev/null +++ b/src/Trace/Reporter/AsyncReporter.php @@ -0,0 +1,116 @@ + 1000, + * 'callPeriod' => 2.0, + * 'workerNum' => 2]. + * @type array $clientConfig Configuration options for the Trace client + * used to handle processing of batch items. + * For valid options please see + * {@see \Google\Cloud\Trace\TraceClient::__construct()}. + * @type BatchRunner $batchRunner A BatchRunner object. Mainly used for + * the tests to inject a mock. **Defaults to** a newly created + * BatchRunner. + * @type string $identifier An identifier for the batch job. + * **Defaults to** `stackdriver-trace`. + * } + */ + public function __construct(array $options = []) + { + $this->setCommonBatchProperties($options + [ + 'identifier' => 'stackdriver-trace', + 'batchMethod' => 'insertBatch' + ]); + self::$client = isset($options['client']) + ? $options['client'] + : new TraceClient($this->clientConfig); + } + + /** + * Report the provided Trace to a backend. + * + * @param TracerInterface $tracer + * @return bool + */ + public function report(TracerInterface $tracer) + { + $spans = $tracer->spans(); + + if (empty($spans)) { + return false; + } + + $trace = self::$client->trace( + $tracer->context()->traceId() + ); + $trace->setSpans($spans); + + try { + return $this->batchRunner->submitItem($this->identifier, $trace); + } catch (\Exception $e) { + return false; + } + } + + /** + * Returns an array representation of a callback which will be used to write + * batch items. + * + * @return array + */ + protected function getCallback() + { + if (!isset(self::$client)) { + self::$client = new TraceClient($this->clientConfig); + } + + return [self::$client, $this->batchMethod]; + } +} diff --git a/src/Trace/RequestHandler.php b/src/Trace/RequestHandler.php index af9b50540f6e..99ac1bb3bcdc 100644 --- a/src/Trace/RequestHandler.php +++ b/src/Trace/RequestHandler.php @@ -30,6 +30,8 @@ * This class manages the logic for sampling and reporting a trace within a * single request. It is not meant to be used directly -- instead, it should * be managed by the RequestTracer as its singleton instance. + * + * @internal */ class RequestHandler { @@ -71,7 +73,7 @@ class RequestHandler private $tracer; /** - * Create a new RequestTracer and start tracing this request. + * Create a new RequestHandler. * * @param ReporterInterface $reporter How to report the trace at the end of the request * @param SamplerInterface $sampler Which sampler to use for sampling requests @@ -115,8 +117,9 @@ public function __construct(ReporterInterface $reporter, SamplerInterface $sampl } /** - * The function registered as the shutdown function. Cleans up the trace and reports using the - * provided ReporterInterface. Adds additional labels to the root span detected from the response. + * The function registered as the shutdown function. Cleans up the trace and + * reports using the provided ReporterInterface. Adds additional labels to + * the root span detected from the response. */ public function onExit() { @@ -152,13 +155,14 @@ public function tracer() } /** - * Instrument a callable by creating a TraceSpan that manages the startTime and endTime. - * If an exception is thrown while executing the callable, the exception will be caught, - * the span will be closed, and the exception will be re-thrown. + * Instrument a callable by creating a TraceSpan that manages the startTime + * and endTime. If an exception is thrown while executing the callable, the + * exception will be caught, the span will be closed, and the exception will + * be re-thrown. * * @param array $spanOptions Options for the span. - * {@see Google\Cloud\Trace\TraceSpan::__construct()} - * @param callable $callable The callable to inSpan. + * {@see Google\Cloud\Trace\TraceSpan::__construct()} + * @param callable $callable The callable to inSpan. * @return mixed Returns whatever the callable returns */ public function inSpan(array $spanOptions, callable $callable, array $arguments = []) @@ -171,7 +175,7 @@ public function inSpan(array $spanOptions, callable $callable, array $arguments * including handling any thrown exceptions. * * @param array $spanOptions [optional] Options for the span. - * {@see Google\Cloud\Trace\TraceSpan::__construct()} + * {@see Google\Cloud\Trace\TraceSpan::__construct()} * @return TraceSpan */ public function startSpan(array $spanOptions = []) diff --git a/src/Trace/Trace.php b/src/Trace/Trace.php index 10cf45a4bdef..307c59edcbca 100644 --- a/src/Trace/Trace.php +++ b/src/Trace/Trace.php @@ -25,7 +25,7 @@ * This plain PHP class represents a Trace resource. For more information see * [TraceResource](https://cloud.google.com/trace/docs/reference/v1/rest/v1/projects.traces#resource-trace) */ -class Trace +class Trace implements \Serializable { use IdGeneratorTrait; use ValidateTrait; @@ -147,7 +147,7 @@ public function spans() * Create an instance of {@see Google\Cloud\Trace\TraceSpan} * * @param array $options [optional] See {@see Google\Cloud\Trace\TraceSpan::__construct()} - * for configuration details. + * for configuration details. * @return TraceSpan */ public function span(array $options = []) @@ -165,4 +165,34 @@ public function setSpans(array $spans) $this->validateBatch($spans, TraceSpan::class); $this->spans = $spans; } + + /** + * Serialize data. + * + * @return string + * @access private + */ + public function serialize() + { + return serialize([ + $this->projectId, + $this->traceId, + $this->spans + ]); + } + + /** + * Unserialize data. + * + * @param string + * @access private + */ + public function unserialize($data) + { + list( + $this->projectId, + $this->traceId, + $this->spans + ) = unserialize($data); + } } diff --git a/src/Trace/TraceClient.php b/src/Trace/TraceClient.php index 2612b99d0d80..b4cf61361cbd 100644 --- a/src/Trace/TraceClient.php +++ b/src/Trace/TraceClient.php @@ -130,7 +130,7 @@ public function insertBatch(array $traces, array $options = []) * point. To see the operations that can be performed on a trace please * see {@see Google\Cloud\Trace\Trace}. If no traceId is provided, one will be * generated for you. - + * * @param string $traceId [optional] The trace id of the trace to reference. * @return Trace */ @@ -140,7 +140,7 @@ public function trace($traceId = null) } /** - * Fetch all traces in the project + * Fetch all traces in the project. * * @see https://cloud.google.com/trace/docs/reference/v1/rest/v1/projects.traces/list Traces list API documentation. * diff --git a/src/Trace/Tracer/ContextTracer.php b/src/Trace/Tracer/ContextTracer.php index f12614dcdf62..b011f4c094ec 100644 --- a/src/Trace/Tracer/ContextTracer.php +++ b/src/Trace/Tracer/ContextTracer.php @@ -18,7 +18,6 @@ namespace Google\Cloud\Trace\Tracer; use Google\Cloud\Core\ArrayTrait; -use Google\Cloud\Trace\TraceClient; use Google\Cloud\Trace\TraceSpan; use Google\Cloud\Trace\TraceContext; @@ -50,7 +49,7 @@ class ContextTracer implements TracerInterface * Create a new ContextTracer * * @param TraceContext $context [optional] The TraceContext to begin with. If none - * provided, a fresh TraceContext will be generated. + * provided, a fresh TraceContext will be generated. */ public function __construct(TraceContext $context = null) { @@ -61,7 +60,7 @@ public function __construct(TraceContext $context = null) * Instrument a callable by creating a Span that manages the startTime and endTime. * * @param array $spanOptions Options for the span. - * {@see Google\Cloud\Trace\TraceSpan::__construct()} + * {@see Google\Cloud\Trace\TraceSpan::__construct()} * @param callable $callable The callable to inSpan. * @param array $arguments [optional] Arguments for the callable. * @return mixed The result of the callable @@ -80,7 +79,7 @@ public function inSpan(array $spanOptions, callable $callable, array $arguments * Start a new Span. The start time is already set to the current time. * * @param array $spanOptions [optional] Options for the span. - * {@see Google\Cloud\Trace\TraceSpan::__construct()} + * {@see Google\Cloud\Trace\TraceSpan::__construct()} */ public function startSpan(array $spanOptions = []) { diff --git a/tests/snippets/Logging/LoggingClientTest.php b/tests/snippets/Logging/LoggingClientTest.php index 48cc81a34a7b..14108d31e4e4 100644 --- a/tests/snippets/Logging/LoggingClientTest.php +++ b/tests/snippets/Logging/LoggingClientTest.php @@ -196,6 +196,19 @@ public function testPsrLogger() $this->assertInstanceOf(PsrLogger::class, $res->returnVal()); } + public function testPsrBatchLogger() + { + $snippet = $this->snippetFromMethod( + LoggingClient::class, + 'psrLogger', + 1 + ); + $snippet->addLocal('logging', $this->client); + + $res = $snippet->invoke('psrLogger'); + $this->assertInstanceOf(PsrLogger::class, $res->returnVal()); + } + public function testLogger() { $snippet = $this->snippetFromMethod(LoggingClient::class, 'logger'); diff --git a/tests/snippets/Logging/PsrLoggerTest.php b/tests/snippets/Logging/PsrLoggerTest.php index a33a505779cb..7d7a124d4fab 100644 --- a/tests/snippets/Logging/PsrLoggerTest.php +++ b/tests/snippets/Logging/PsrLoggerTest.php @@ -49,6 +49,13 @@ public function testClass() $this->assertInstanceOf(PsrLogger::class, $res->returnVal()); } + public function testClassBatch() + { + $snippet = $this->snippetFromClass(PsrLogger::class, 1); + $res = $snippet->invoke('psrLogger'); + $this->assertInstanceOf(PsrLogger::class, $res->returnVal()); + } + public function testEmergency() { $snippet = $this->snippetFromMethod(PsrLogger::class, 'emergency'); diff --git a/tests/snippets/PubSub/BatchPublisherTest.php b/tests/snippets/PubSub/BatchPublisherTest.php new file mode 100644 index 000000000000..8ff1429f22d7 --- /dev/null +++ b/tests/snippets/PubSub/BatchPublisherTest.php @@ -0,0 +1,64 @@ +batchPublisher = $this->prophesize(BatchPublisher::class); + $this->batchPublisher->publish([ + 'data' => 'An important message.' + ]) + ->willReturn(true); + } + + public function testClass() + { + $snippet = $this->snippetFromClass(BatchPublisher::class); + $topic = $this->prophesize(Topic::class); + $topic->batchPublisher() + ->willReturn($this->batchPublisher->reveal()); + $pubsub = $this->prophesize(PubSubClient::class); + $pubsub->topic(Argument::type('string')) + ->willReturn($topic->reveal()); + $snippet->setLine(2, ''); + $snippet->addLocal('pubsub', $pubsub->reveal()); + + $snippet->invoke(); + } + + public function testPublish() + { + $snippet = $this->snippetFromMethod(BatchPublisher::class, 'publish'); + $snippet->addLocal('batchPublisher', $this->batchPublisher->reveal()); + + $snippet->invoke(); + } +} diff --git a/tests/snippets/PubSub/TopicTest.php b/tests/snippets/PubSub/TopicTest.php index 64e3d3595d7b..c776699f9b04 100644 --- a/tests/snippets/PubSub/TopicTest.php +++ b/tests/snippets/PubSub/TopicTest.php @@ -20,6 +20,7 @@ use Google\Cloud\Core\Iam\Iam; use Google\Cloud\Dev\Snippet\SnippetTestCase; use Google\Cloud\PubSub\Connection\ConnectionInterface; +use Google\Cloud\PubSub\BatchPublisher; use Google\Cloud\PubSub\PubSubClient; use Google\Cloud\PubSub\Subscription; use Google\Cloud\PubSub\Topic; @@ -182,6 +183,20 @@ public function testPublishBatch() $snippet->invoke(); } + public function testBatchPublisher() + { + $snippet = $this->snippetFromMethod(Topic::class, 'batchPublisher'); + $batchPublisher = $this->prophesize(BatchPublisher::class); + $batchPublisher->publish(Argument::type('array')) + ->willReturn(true); + $topic = $this->prophesize(Topic::class); + $topic->batchPublisher() + ->willReturn($batchPublisher->reveal()); + $snippet->addLocal('topic', $topic->reveal()); + + $snippet->invoke(); + } + public function testSubscribe() { $snippet = $this->snippetFromMethod(Topic::class, 'subscribe'); diff --git a/tests/system/Core/Batch/BatchRunnerTest.php b/tests/system/Core/Batch/BatchRunnerTest.php new file mode 100644 index 000000000000..ccd28bfa40fb --- /dev/null +++ b/tests/system/Core/Batch/BatchRunnerTest.php @@ -0,0 +1,168 @@ + array('file', 'php://stdin', 'r'), + 1 => array('file', 'php://stdout', 'w'), + 1 => array('file', 'php://stderr', 'w') + ); + self::$daemon = proc_open( + $daemon_command, + $descriptorSpec, + $pipes + ); + putenv('IS_BATCH_DAEMON_RUNNING=true'); + } else { + // Use in-memory implementation. + putenv('IS_BATCH_DAEMON_RUNNING'); + } + } + + public static function tearDownAfterClass() + { + @proc_terminate(self::$daemon); + @proc_close(self::$daemon); + @unlink(self::$targetFile); + @unlink(self::$commandFile); + self::delTree(self::$testDir); + putenv('GOOGLE_CLOUD_BATCH_DAEMON_FAILURE_DIR'); + putenv('IS_BATCH_DAEMON_RUNNING'); + } + + public function setup() + { + $this->runner = new BatchRunner(); + $myJob = new MyJob(self::$commandFile, self::$targetFile); + $this->runner->registerjob( + 'batch-daemon-system-test', + array($myJob, 'runJob'), + array( + 'workerNum' => 1, + 'batchSize' => 2, + 'callPeriod' => 1, + ) + ); + } + + public function getResult() + { + usleep(100000); + return file_get_contents(self::$targetFile); + } + + public function assertResultContains($expected) + { + $this->assertContains($expected, $this->getResult()); + } + + public function testSubmit() + { + $this->runner->submitItem('batch-daemon-system-test', 'apple'); + // It should be still in the buffer. + $this->assertEmpty($this->getResult()); + $this->runner->submitItem('batch-daemon-system-test', 'orange'); + $this->assertResultContains('APPLE'); + $this->assertResultContains('ORANGE'); + + // This item should be picked by the call period. + sleep(1); + $this->runner->submitItem('batch-daemon-system-test', 'peach'); + $this->assertResultContains('PEACH'); + + // Failure simulation + file_put_contents(self::$commandFile, 'fail'); + + $this->runner->submitItem('batch-daemon-system-test', 'banana'); + $this->runner->submitItem('batch-daemon-system-test', 'lemon'); + $result = $this->getResult(); + $this->assertNotContains('BANANA', $result); + $this->assertNotContains('LEMON', $result); + + // Retry simulation + unlink(self::$commandFile); + if (self::$run_daemon) { + $retry_command = __DIR__ + . '/../../../../src/Core/bin/google-cloud-batch retry'; + exec($retry_command); + } else { + // The in-memory implementation doesn't share the BatchConfig with + // other processes, so we need to run retryAll in the same process. + $retry = new Retry(); + $retry->retryAll(); + } + $this->assertResultContains('BANANA'); + $this->assertResultContains('LEMON'); + } +} diff --git a/tests/system/Core/Batch/MyJob.php b/tests/system/Core/Batch/MyJob.php new file mode 100644 index 000000000000..7be1b5ba9104 --- /dev/null +++ b/tests/system/Core/Batch/MyJob.php @@ -0,0 +1,54 @@ +commandFile = $commandFile; + $this->targetFile = $targetFile; + } + + /** + * A method that we use for the test. It will return false when + * the command file contains 'fail'. + */ + public function runJob($items) + { + $failCommand = @file_get_contents($this->commandFile); + if ($failCommand === 'fail') { + return false; + } + $fp = fopen($this->targetFile, 'w+'); + if (flock($fp, LOCK_EX)) { + foreach ($items as $item) { + fwrite($fp, strtoupper($item) . PHP_EOL); + } + } else { + echo 'Could not get the lock'; + @fclose($fp); + return false; + } + fclose($fp); + return true; + } +} diff --git a/tests/unit/Core/Batch/BatchConfigTest.php b/tests/unit/Core/Batch/BatchConfigTest.php new file mode 100644 index 000000000000..f482fc619fff --- /dev/null +++ b/tests/unit/Core/Batch/BatchConfigTest.php @@ -0,0 +1,90 @@ +config = new BatchConfig(); + $this->identifier = 'job1'; + $this->func = 'myFunc'; + $this->config->registerJob( + $this->identifier, + $this->func, + [] + ); + // It must have 1 as the idNum. + $this->idNum = 1; + } + + public function testGetJobFromId() + { + $job = $this->config->getJobFromId($this->identifier); + $this->assertEquals($this->idNum, $job->getIdNum()); + $this->assertEquals($this->identifier, $job->getIdentifier()); + $this->assertNull($this->config->getJobFromId('bogus')); + } + + public function testGetJobFromIdNum() + { + $job = $this->config->getJobFromIdNum($this->idNum); + $this->assertEquals($this->idNum, $job->getIdNum()); + $this->assertEquals($this->identifier, $job->getIdentifier()); + $this->assertNull($this->config->getJobFromIdNum(10)); + } + + public function testRegisterJob() + { + $identifier = 'job2'; + $this->config->registerJob( + $identifier, + $this->func, + [] + ); + // The idNum is 1 origin, incremented by 1 + $job = $this->config->getJobFromIdNum(2); + $this->assertEquals(2, $job->getIdNum()); + $this->assertEquals($identifier, $job->getIdentifier()); + } + + public function testGetjobs() + { + $identifier = 'job2'; + $this->config->registerJob( + $identifier, + $this->func, + [] + ); + $jobs = $this->config->getJobs(); + $this->assertEquals(count($jobs), 2); + $this->assertEquals($this->idNum, $jobs[$this->identifier]->getIdNum()); + $this->assertEquals(2, $jobs[$identifier]->getIdNum()); + } +} diff --git a/tests/unit/Core/Batch/BatchJobTest.php b/tests/unit/Core/Batch/BatchJobTest.php new file mode 100644 index 000000000000..9c28edb2ea38 --- /dev/null +++ b/tests/unit/Core/Batch/BatchJobTest.php @@ -0,0 +1,81 @@ +assertEquals(100, $job->getBatchSize()); + $this->assertEquals(2.0, $job->getCallPeriod()); + $this->assertEquals(1, $job->getWorkerNum()); + $this->assertNull($job->getBootstrapFile()); + $this->assertEquals(1, $job->getIdNum()); + $this->assertEquals('testing', $job->getIdentifier()); + } + + public function testCustom() + { + $job = new BatchJob( + 'testing', + array($this, 'runJob'), + 1, + array( + 'batchSize' => 1000, + 'callPeriod' => 1.0, + 'bootstrapFile' => __FILE__, + 'workerNum' => 10 + ) + ); + $this->assertEquals(1000, $job->getBatchSize()); + $this->assertEquals(1.0, $job->getCallPeriod()); + $this->assertEquals(10, $job->getWorkerNum()); + $this->assertEquals(__FILE__, $job->getBootstrapFile()); + $this->assertEquals(1, $job->getIdNum()); + $this->assertEquals('testing', $job->getIdentifier()); + } + + public function testRun() + { + $job = new BatchJob('testing', array($this, 'runJob'), 1); + $items = array('apple', 'orange', 'banana'); + $expected = array('APPLE', 'ORANGE', 'BANANA'); + $job->run($items); + $this->assertEquals($expected, $this->items); + } + + /** + * A method that we use for the test. + */ + public function runJob($items) + { + foreach ($items as $item) { + $this->items[] = strtoupper($item); + } + return true; + } +} diff --git a/tests/unit/Core/Batch/BatchRunnerTest.php b/tests/unit/Core/Batch/BatchRunnerTest.php new file mode 100644 index 000000000000..c7456754bb24 --- /dev/null +++ b/tests/unit/Core/Batch/BatchRunnerTest.php @@ -0,0 +1,139 @@ +configStorage = $this->prophesize(ConfigStorageInterface::class); + $this->processor = $this->prophesize(ProcessItemInterface::class); + $this->batchConfig = $this->prophesize(BatchConfig::class); + } + + /** + * @expectedException \InvalidArgumentException + */ + public function testRegisterJobClosure() + { + $runner = new BatchRunner( + $this->configStorage->reveal(), + $this->processor->reveal() + ); + $result = $runner->registerJob( + 'test', + function() {} + ); + $this->assertTrue(false, 'It should throw InvalidArgumentException'); + } + + public function testConstructorLoadConfig() + { + $job = new BatchJob('test', 'myFunc', 1); + $this->batchConfig->getJobFromIdNum(1) + ->willReturn($job) + ->shouldBeCalledTimes(1); + $this->batchConfig->getJobFromId('test') + ->willReturn($job) + ->shouldBeCalledTimes(1); + $this->batchConfig->getJobs() + ->willReturn(array('test' => $job)) + ->shouldBeCalledTimes(1); + $config = $this->batchConfig->reveal(); + $this->configStorage->lock() + ->willreturn(true) + ->shouldBeCalledTimes(1); + $this->configStorage->load() + ->willreturn($config) + ->shouldBeCalledTimes(1); + $this->configStorage->unlock() + ->willreturn(true) + ->shouldBeCalledTimes(1); + $runner = new BatchRunner( + $this->configStorage->reveal(), + $this->processor->reveal() + ); + $this->assertEquals($job, $runner->getJobFromIdNum(1)); + $this->assertEquals($job, $runner->getJobFromId('test')); + $this->assertEquals(array('test' => $job), $runner->getJobs()); + } + + public function testRegisterJob() + { + $this->batchConfig->registerJob('test', 'myFunc', []) + ->shouldBeCalledTimes(1); + $config = $this->batchConfig->reveal(); + $this->configStorage->lock() + ->willreturn(true) + ->shouldBeCalledTimes(2); + $this->configStorage->load() + ->willreturn($config) + ->shouldBeCalledTimes(2); + $this->configStorage->save(Argument::type(BatchConfig::class)) + ->willreturn(true) + ->shouldBeCalledTimes(1); + $this->configStorage->unlock() + ->willreturn(true) + ->shouldBeCalledTimes(2); + $runner = new BatchRunner( + $this->configStorage->reveal(), + $this->processor->reveal() + ); + $result = $runner->registerJob('test', 'myFunc'); + $this->assertTrue($result); + } + + public function testSubmitItem() + { + $job = new BatchJob('test', 'myFunc', 1); + $this->batchConfig->getJobFromId('test') + ->willReturn($job) + ->shouldBeCalledTimes(1); + $config = $this->batchConfig->reveal(); + $this->configStorage->lock() + ->willreturn(true) + ->shouldBeCalledTimes(1); + $this->configStorage->load() + ->willreturn($config) + ->shouldBeCalledTimes(1); + $this->configStorage->unlock() + ->willreturn(true) + ->shouldBeCalledTimes(1); + $this->processor->submit('item', 1) + ->shouldBeCalledTimes(1); + $runner = new BatchRunner( + $this->configStorage->reveal(), + $this->processor->reveal() + ); + $runner->submitItem('test', 'item'); + } +} diff --git a/tests/unit/Core/Batch/BatchTraitTest.php b/tests/unit/Core/Batch/BatchTraitTest.php new file mode 100644 index 000000000000..4c6df330255f --- /dev/null +++ b/tests/unit/Core/Batch/BatchTraitTest.php @@ -0,0 +1,150 @@ +prophesize(BatchJob::class); + $job->getIdNum() + ->willReturn($idNum) + ->shouldBeCalledTimes(1); + $processor = $this->prophesize(ProcessItemInterface::class); + $processor->flush($idNum) + ->willReturn($returnVal); + $runner = $this->prophesize(BatchRunner::class); + $runner->getJobFromId(Argument::any()) + ->willReturn($job->reveal()) + ->shouldBeCalledTimes(1); + $runner->getProcessor() + ->willReturn($processor->reveal()) + ->shouldBeCalledTimes(1); + $impl = new BatchClass(['batchRunner' => $runner->reveal()]); + + $this->assertEquals($returnVal, $impl->flush()); + } + + public function testSend() + { + $items = ['a', 'b', 'c']; + $temp = fopen('php://temp', 'rw'); + $hasExecuted = false; + $count = 0; + $impl = new BatchClass([ + 'debugOutput' => true, + 'debugOutputResource' => $temp, + 'cb' => function (array $items) use (&$count, &$hasExecuted) { + $hasExecuted = true; + $count = count($items); + } + ]); + + $impl->send($items); + + rewind($temp); + $contents = stream_get_contents($temp); + + $this->assertTrue($hasExecuted); + $this->assertEquals(count($items), $count); + $this->assertContains('seconds for ' . self::BATCH_METHOD, $contents); + } + + /** + * @expectedException \InvalidArgumentException + */ + public function testSetCommonBatchPropertiesThrowsExceptionWithoutIdentifier() + { + $impl = new BatchClass(); + $impl->setCommonBatchProperties(['batchMethod' => self::BATCH_METHOD]); + } + + /** + * @expectedException \InvalidArgumentException + */ + public function testSetCommonBatchPropertiesThrowsExceptionWithoutBatchMethod() + { + $impl = new BatchClass(); + $impl->setCommonBatchProperties(['identifier' => self::ID]); + } +} + +class BatchClass +{ + use BatchTrait { + flush as publicFlush; + send as publicSend; + setCommonBatchProperties as privateSetCommonBatchProperties; + } + + private $cb; + + public function __construct(array $options = []) + { + $options += [ + 'batchRunner' => null, + 'identifier' => BatchTraitTest::ID, + 'batchMethod' => BatchTraitTest::BATCH_METHOD, + 'debugOutput' => false, + 'debugOutputResource' => null, + 'cb' => null + ]; + + $this->batchRunner = $options['batchRunner']; + $this->identifier = $options['identifier']; + $this->batchMethod = $options['batchMethod']; + $this->debugOutput = $options['debugOutput']; + $this->debugOutputResource = $options['debugOutputResource']; + $this->cb = $options['cb']; + } + + function flush() + { + return $this->publicFlush(); + } + + function send(array $items) + { + return $this->publicSend($items); + } + + function setCommonBatchProperties(array $options) + { + $this->privateSetCommonBatchProperties($options); + } + + function getCallback() + { + return $this->cb; + } +} diff --git a/tests/unit/Core/Batch/HandleFailureTraitTest.php b/tests/unit/Core/Batch/HandleFailureTraitTest.php new file mode 100644 index 000000000000..29eb607b6d2a --- /dev/null +++ b/tests/unit/Core/Batch/HandleFailureTraitTest.php @@ -0,0 +1,139 @@ +delTree($target) : unlink($target); + } + return rmdir($dir); + } + + public function setUp() + { + $this->impl = new HandleFailureClass(); + $this->testDir = sprintf( + '%s/google-cloud-unit-test-%d', + sys_get_temp_dir(), + getmypid() + ); + @mkdir($this->testDir); + putenv('GOOGLE_CLOUD_BATCH_DAEMON_FAILURE_DIR'); + } + + public function tearDown() + { + $this->delTree($this->testDir); + putenv('GOOGLE_CLOUD_BATCH_DAEMON_FAILURE_DIR'); + } + + /** + * @ExpectedException \RuntimeException + */ + public function testInitFailureFileThrowsException() + { + putenv( + 'GOOGLE_CLOUD_BATCH_DAEMON_FAILURE_DIR=/tmp/non-existent/subdir'); + $this->impl->initFailureFile(); + } + + public function testInitFailureFile() + { + $this->impl->initFailureFile(); + $this->assertEquals( + sprintf('%s/batch-daemon-failure', sys_get_temp_dir()), + $this->impl->getBaseDir() + ); + $this->assertEquals( + sprintf( + '%s/failed-items-%d', + $this->impl->getBaseDir(), + getmypid() + ), + $this->impl->getFailureFile() + ); + putenv('GOOGLE_CLOUD_BATCH_DAEMON_FAILURE_DIR=/tmp'); + $this->impl->initFailureFile(); + $this->assertEquals('/tmp', $this->impl->getBaseDir()); + $this->assertEquals( + sprintf( + '%s/failed-items-%d', + $this->impl->getBaseDir(), + getmypid() + ), + $this->impl->getFailureFile() + ); + } + + public function testHandleFailure() + { + putenv('GOOGLE_CLOUD_BATCH_DAEMON_FAILURE_DIR=' . $this->testDir); + $this->impl->initFailureFile(); + $this->impl->handleFailure(1, array('apple', 'orange')); + $files = $this->impl->getFailedFiles(); + $this->assertCount(1, $files); + $unserialized = unserialize(file_get_contents($files[0])); + $this->assertEquals( + array(1 => array('apple', 'orange')), + $unserialized + ); + } +} + +class HandleFailureClass +{ + use HandleFailureTrait { + initFailureFile as privateInitFailureFile; + getFailedFiles as privateGetFailedFiles; + } + + public function getFailureFile() + { + return $this->failureFile; + } + + public function getBaseDir() + { + return $this->baseDir; + } + + public function initFailureFile() + { + return $this->privateInitFailureFile(); + } + + public function getFailedFiles() + { + return $this->privateGetFailedFiles(); + } +} diff --git a/tests/unit/Core/Batch/InMemoryConfigStorageTest.php b/tests/unit/Core/Batch/InMemoryConfigStorageTest.php new file mode 100644 index 000000000000..76293c0012b4 --- /dev/null +++ b/tests/unit/Core/Batch/InMemoryConfigStorageTest.php @@ -0,0 +1,111 @@ +assertEquals($configStorage1, $configStorage2); + } + + public function testConstructorIsForbidden() + { + $reflection = new \ReflectionClass( + '\Google\Cloud\Core\Batch\InMemoryConfigStorage'); + $constructor = $reflection->getConstructor(); + $this->assertFalse($constructor->isPublic()); + } + + public function testLock() + { + $configStorage = InMemoryConfigStorage::getInstance(); + $result = $configStorage->lock(); + $this->assertTrue($result); + } + + public function testUnLock() + { + $configStorage = InMemoryConfigStorage::getInstance(); + $result = $configStorage->unlock(); + $this->assertTrue($result); + } + + public function testSaveAndLoad() + { + $configStorage = InMemoryConfigStorage::getInstance(); + $config = new BatchConfig(); + $configStorage->save($config); + $this->assertEquals($config, $configStorage->load()); + } + + public function testSubmit() + { + $configStorage = InMemoryConfigStorage::getInstance(); + $config = new BatchConfig(); + $config->registerJob( + 'testSubmit', + array($this, 'runJob'), + array('batchSize' => 2) + ); + $configStorage->save($config); + + $configStorage->submit('apple', 1); + // The job hasn't been run because of the batchSize. + $this->assertEmpty($this->items); + $configStorage->submit('orange', 1); + $this->assertEquals( + array('APPLE', 'ORANGE'), + $this->items + ); + $configStorage->submit('banana', 1); + // It's in the buffer. + $this->assertEquals( + array('APPLE', 'ORANGE'), + $this->items + ); + // shutdown will pick the remainder. + $configStorage->shutdown(); + $this->assertEquals( + array('APPLE', 'ORANGE', 'BANANA'), + $this->items + ); + } + + /** + * A method that we use for the test. + */ + public function runJob($items) + { + foreach ($items as $item) { + $this->items[] = strtoupper($item); + } + return true; + } +} diff --git a/tests/unit/Core/Batch/RetryTest.php b/tests/unit/Core/Batch/RetryTest.php new file mode 100644 index 000000000000..139ab26541e5 --- /dev/null +++ b/tests/unit/Core/Batch/RetryTest.php @@ -0,0 +1,97 @@ +runner = $this->prophesize(BatchRunner::class); + $this->job = $this->prophesize(BatchJob::class); + } + + public function testRetryAll() + { + $this->job->run(array('apple', 'orange')) + ->willReturn(true) + ->shouldBeCalledTimes(1); + $this->runner->getJobFromIdNum(1) + ->willReturn($this->job->reveal()) + ->shouldBeCalledTimes(1); + $this->retry = new Retry($this->runner->reveal()); + $this->retry->handleFailure(1, array('apple', 'orange')); + $this->assertEquals(1, count(glob(self::$testDir . '/failed-items*'))); + $this->retry->retryAll(); + $this->assertEquals(0, count(glob(self::$testDir . '/failed-items*'))); + } + + public function testRetryAllWithSingleFailure() + { + $this->job->run(array('apple', 'orange')) + ->willReturn(false, true) + ->shouldBeCalledTimes(2); + $this->runner->getJobFromIdNum(1) + ->willReturn($this->job->reveal()) + ->shouldBeCalledTimes(2); + $this->retry = new Retry($this->runner->reveal()); + $this->retry->handleFailure(1, array('apple', 'orange')); + $this->retry->retryAll(); + $this->assertEquals(1, count(glob(self::$testDir . '/failed-items*'))); + $this->retry->retryAll(); + $this->assertEquals(0, count(glob(self::$testDir . '/failed-items*'))); + } +} diff --git a/tests/unit/Core/Batch/SysvConfigStorageTest.php b/tests/unit/Core/Batch/SysvConfigStorageTest.php new file mode 100644 index 000000000000..23b95da2ed99 --- /dev/null +++ b/tests/unit/Core/Batch/SysvConfigStorageTest.php @@ -0,0 +1,55 @@ +isSysvIPCLOaded()) { + $this->markTestSkipped( + 'Skipping because SystemV IPC extensions are not loaded'); + } + $this->storage = new SysvConfigStorage(); + } + + public function testLockAndUnlock() + { + $this->assertTrue($this->storage->lock()); + $this->assertTrue($this->storage->unlock()); + } + + public function testSaveAndLoad() + { + $config = new BatchConfig(); + $this->storage->save($config); + $this->assertEquals($config, $this->storage->load()); + } +} diff --git a/tests/unit/Core/Batch/SysvProcessorTest.php b/tests/unit/Core/Batch/SysvProcessorTest.php new file mode 100644 index 000000000000..364ba4692241 --- /dev/null +++ b/tests/unit/Core/Batch/SysvProcessorTest.php @@ -0,0 +1,84 @@ +isSysvIPCLOaded()) { + $this->markTestSkipped( + 'Skipping because SystemV IPC extensions are not loaded'); + } + $this->processor = new SysvProcessor(); + } + + public function tearDown() + { + putenv('GOOGLE_CLOUD_SYSV_ID'); + } + + /** + * @dataProvider items + */ + public function testSubmit($item, $exptectedType) + { + $this->processor->submit($item, 1); + $q = msg_get_queue($this->getSysvKey(1)); + $result = msg_receive( + $q, + 0, + $type, + 8192, + $message, + true, + MSG_IPC_NOWAIT, + $errorcode + ); + $this->assertTrue($result); + $this->assertEquals($exptectedType, $type); + if ($type === self::$typeDirect) { + $this->assertEquals($item, $message); + } else { + $this->assertEquals( + $item, + unserialize(file_get_contents($message))); + @unlink($message); + } + } + + public function items() + { + return [ + ['item', self::$typeDirect], + [str_repeat('x', 8193), self::$typeFile] + ]; + } +} diff --git a/tests/unit/Core/Batch/SysvTraitTest.php b/tests/unit/Core/Batch/SysvTraitTest.php new file mode 100644 index 000000000000..bd0b3509c9ea --- /dev/null +++ b/tests/unit/Core/Batch/SysvTraitTest.php @@ -0,0 +1,93 @@ +impl = new MySysvClass(); + } + + public function testGetSysvKey() + { + if (! $this->impl->isSysvIPCLoaded()) { + $this->markTestSkipped( + 'SysV IPC extensions are not available, skipped'); + } + $key1 = $this->impl->getSysvKey(1); + $key2 = $this->impl->getSysvKey(2); + $this->assertEquals(1, $key2 - $key1); + } + + public function testIsSysvIPCLoaded() + { + $expected = extension_loaded('sysvmsg') + && extension_loaded('sysvsem') + && extension_loaded('sysvshm'); + $this->assertEquals($expected, $this->impl->isSysvIPCLoaded()); + } + + public function testIsDaemonRunning() + { + // Clear the env + $orig = getenv('IS_BATCH_DAEMON_RUNNING'); + try { + putenv('IS_BATCH_DAEMON_RUNNING'); + $this->assertFalse($this->impl->isDaemonRunning()); + putenv('IS_BATCH_DAEMON_RUNNING=true'); + $this->assertTrue($this->impl->isDaemonRunning()); + } finally { + if ($orig === false) { + putenv('IS_BATCH_DAEMON_RUNNING'); + } else { + putenv('IS_BATCH_DAEMON_RUNNING=' . $orig); + } + } + } +} + +class MySysvClass +{ + use SysvTrait { + isDaemonRunning as privateIsDaemonRunning; + isSysvIPCLoaded as privateIsSysvIPCLoaded; + getSysvKey as privateGetSysvKey; + } + + function isDaemonRunning() + { + return $this->privateIsDaemonRunning(); + } + + function isSysvIPCLoaded() + { + return $this->privateIsSysvIPCLoaded(); + } + + function getSysvKey($id) + { + return $this->privateGetSysvKey($id); + } +} diff --git a/tests/unit/Core/ClientTraitTest.php b/tests/unit/Core/ClientTraitTest.php index 255c8eb23bbf..54fa555ddc2e 100644 --- a/tests/unit/Core/ClientTraitTest.php +++ b/tests/unit/Core/ClientTraitTest.php @@ -157,6 +157,28 @@ public function testDetectProjectIdWithNoProjectIdAvailable() ]); } + public function testProjectIdFromEnv() + { + $projectId = 'project-from-env'; + + $trait = new ClientTraitStub(); + + $originalEnv = getenv('GCLOUD_PROJECT'); + + try { + putenv('GCLOUD_PROJECT=' . $projectId); + $res = $trait->runDetectProjectId([]); + + $this->assertEquals($res, $projectId); + } finally { + if ($originalEnv === false) { + putenv('GCLOUD_PROJECT'); + } else { + putenv('GCLOUD_PROJECT=' . $originalEnv); + } + } + } + public function testDetectProjectIdOnGce() { $projectId = 'gce-project-rawks'; diff --git a/tests/unit/Core/Report/EmptyMetadataProviderTest.php b/tests/unit/Core/Report/EmptyMetadataProviderTest.php new file mode 100644 index 000000000000..b69dc12165b0 --- /dev/null +++ b/tests/unit/Core/Report/EmptyMetadataProviderTest.php @@ -0,0 +1,62 @@ +metadataProvider = new EmptyMetadataProvider(); + } + + public function testMonitoredResource() + { + $this->assertEquals( + [], + $this->metadataProvider->monitoredResource() + ); + } + + public function testProjectId() + { + $this->assertEquals('', $this->metadataProvider->projectId()); + } + + public function testServiceId() + { + $this->assertEquals('', $this->metadataProvider->serviceId()); + } + + public function testVersionId() + { + $this->assertEquals('', $this->metadataProvider->versionId()); + } + + public function testLabels() + { + $this->assertEquals([], $this->metadataProvider->labels()); + } +} + diff --git a/tests/unit/Core/Report/GAEFlexMetadataProviderTest.php b/tests/unit/Core/Report/GAEFlexMetadataProviderTest.php new file mode 100644 index 000000000000..dac8fd5b292b --- /dev/null +++ b/tests/unit/Core/Report/GAEFlexMetadataProviderTest.php @@ -0,0 +1,81 @@ + 'my-service', + 'GAE_VERSION' => 'my-version', + 'GCLOUD_PROJECT' => 'my-project', + 'HTTP_X_CLOUD_TRACE_CONTEXT' => 'my-traceId' + ]; + + public function testWithEnvs() + { + $metadataProvider = new GAEFlexMetadataProvider($this->envs); + $this->assertEquals( + [ + 'type' => 'gae_app', + 'labels' => [ + 'project_id' => 'my-project', + 'version_id' => 'my-version', + 'module_id' => 'my-service' + ] + ], + $metadataProvider->monitoredResource() + ); + $this->assertEquals('my-project', $metadataProvider->projectId()); + $this->assertEquals('my-service', $metadataProvider->serviceId()); + $this->assertEquals('my-version', $metadataProvider->versionId()); + $this->assertEquals( + [ + 'appengine.googleapis.com/trace_id' => 'my-traceId' + ], + $metadataProvider->labels() + ); + } + + public function testWithOutEnvs() + { + $metadataProvider = new GAEFlexMetadataProvider([]); + $this->assertEquals( + [ + 'type' => 'gae_app', + 'labels' => [ + 'project_id' => 'unknown-projectid', + 'version_id' => 'unknown-version', + 'module_id' => 'unknown-service' + ] + ], + $metadataProvider->monitoredResource() + ); + $this->assertEquals( + 'unknown-projectid', + $metadataProvider->projectId() + ); + $this->assertEquals('unknown-service', $metadataProvider->serviceId()); + $this->assertEquals('unknown-version', $metadataProvider->versionId()); + $this->assertEquals([], $metadataProvider->labels()); + } +} diff --git a/tests/unit/Core/Report/MetadataProviderUtilsTest.php b/tests/unit/Core/Report/MetadataProviderUtilsTest.php new file mode 100644 index 000000000000..dc3a9965d5ef --- /dev/null +++ b/tests/unit/Core/Report/MetadataProviderUtilsTest.php @@ -0,0 +1,44 @@ + 'my-service']; + + public function testAutoSelect() + { + $metadataProvider = MetadataProviderUtils::autoSelect($this->envs); + $this->assertInstanceOf( + GaeFlexMetadataProvider::class, + $metadataProvider + ); + $metadataProvider = MetadataProviderUtils::autoSelect([]); + $this->assertInstanceOf( + EmptyMetadataProvider::class, + $metadataProvider + ); + } +} diff --git a/tests/unit/Core/Report/SimpleMetadataProviderTest.php b/tests/unit/Core/Report/SimpleMetadataProviderTest.php new file mode 100644 index 000000000000..2f643f4fe909 --- /dev/null +++ b/tests/unit/Core/Report/SimpleMetadataProviderTest.php @@ -0,0 +1,90 @@ + 'gae_app']; + + private $projectId = 'my-project'; + + private $serviceId = 'my-service'; + + private $versionId = 'my-version'; + + private $labels = ['key' => 'value']; + + public function setup() + { + $this->metadataProvider = new SimpleMetadataProvider( + $this->monitoredResource, + $this->projectId, + $this->serviceId, + $this->versionId, + $this->labels + ); + } + + public function testGetMonitoredResource() + { + $this->assertEquals( + $this->monitoredResource, + $this->metadataProvider->monitoredResource() + ); + } + + public function testProjectId() + { + $this->assertEquals( + $this->projectId, + $this->metadataProvider->projectId() + ); + } + + public function testService() + { + $this->assertEquals( + $this->serviceId, + $this->metadataProvider->serviceId() + ); + } + + public function testVersion() + { + $this->assertEquals( + $this->versionId, + $this->metadataProvider->versionId() + ); + } + + public function testLabels() + { + $this->assertEquals( + $this->labels, + $this->metadataProvider->labels() + ); + } +} + diff --git a/tests/unit/ErrorReporting/BootstrapTest.php b/tests/unit/ErrorReporting/BootstrapTest.php new file mode 100644 index 000000000000..a27fcf5ea867 --- /dev/null +++ b/tests/unit/ErrorReporting/BootstrapTest.php @@ -0,0 +1,298 @@ +psrBatchLogger = $this->prophesize(PsrLogger::class); + } + + /** + * @dataProvider levelAndErrorPrefixProvider + */ + public function testGetErrorPrefix($level, $expectedPrefix) + { + $prefix = Bootstrap::getErrorPrefix($level); + $this->assertEquals($expectedPrefix, $prefix); + } + + public function levelAndErrorPrefixProvider() + { + return [ + [E_PARSE, 'PHP Parse error'], + [E_ERROR, 'PHP Fatal error'], + [E_CORE_ERROR, 'PHP Fatal error'], + [E_COMPILE_ERROR, 'PHP Fatal error'], + [E_USER_ERROR, 'PHP error'], + [E_RECOVERABLE_ERROR, 'PHP error'], + [E_WARNING, 'PHP Warning'], + [E_CORE_WARNING, 'PHP Warning'], + [E_COMPILE_WARNING, 'PHP Warning'], + [E_USER_WARNING, 'PHP Warning'], + [E_NOTICE, 'PHP Notice'], + [E_USER_NOTICE, 'PHP Notice'], + [E_STRICT, 'PHP Debug'], + [PHP_INT_MAX, 'PHP Notice'], + ]; + } + + /** + * @dataProvider levelAndErrorLevelString + */ + public function testGetErrorLevelString($level, $expectedErrorLevelString) + { + $errorLevelString = Bootstrap::getErrorLevelString($level); + $this->assertEquals($expectedErrorLevelString, $errorLevelString); + } + + public function levelAndErrorLevelString() + { + return [ + [E_PARSE, 'CRITICAL'], + [E_ERROR, 'ERROR'], + [E_CORE_ERROR, 'ERROR'], + [E_COMPILE_ERROR, 'ERROR'], + [E_USER_ERROR, 'ERROR'], + [E_RECOVERABLE_ERROR, 'ERROR'], + [E_WARNING, 'WARNING'], + [E_CORE_WARNING, 'WARNING'], + [E_COMPILE_WARNING, 'WARNING'], + [E_USER_WARNING, 'WARNING'], + [E_NOTICE, 'NOTICE'], + [E_USER_NOTICE, 'NOTICE'], + [E_STRICT, 'DEBUG'], + [PHP_INT_MAX, 'NOTICE'], + ]; + } + + /** + * @dataProvider exceptionProvider + */ + public function testExceptionHandler( + $exception + ) { + $expectedMessage = sprintf('PHP Notice: %s', (string)$exception); + $this->psrBatchLogger->error($expectedMessage) + ->shouldBeCalledTimes(1); + Bootstrap::$psrBatchLogger = $this->psrBatchLogger->reveal(); + Bootstrap::exceptionHandler($exception); + } + + /** + * @dataProvider exceptionProvider + */ + public function testExceptionHandlerWithoutLogger( + $exception + ) { + $expectedMessage = sprintf('PHP Notice: %s', (string)$exception); + Bootstrap::$psrBatchLogger = null; + Bootstrap::exceptionHandler($exception); + $this->assertEquals($expectedMessage . PHP_EOL, MockValues::$stderr); + } + + public function exceptionProvider() + { + return [ + [new \Exception('My awesome exception')] + ]; + } + + /** + * @dataProvider errorsAndMetadataProvider + */ + public function testErrorHandler( + $error, + $resource, + $projectId, + $serviceId, + $versionId + ) { + $metadataProvider = new SimpleMetadataProvider( + $resource, + $projectId, + $serviceId, + $versionId + ); + $this->psrBatchLogger->getMetadataProvider() + ->willReturn($metadataProvider) + ->shouldBeCalledTimes(2); + $expectedMessage = sprintf( + '%s: %s in %s on line %d', + Bootstrap::getErrorPrefix($error['type']), + $error['message'], + $error['file'], + $error['line'] + ); + $expectedContext = [ + 'context' => [ + 'reportLocation' => [ + 'filePath' => $error['file'], + 'lineNumber' => $error['line'], + 'functionName' => 'unknown' + ] + ], + 'serviceContext' => [ + 'service' => $serviceId, + 'version' => $versionId, + ] + ]; + $this->psrBatchLogger->log( + Bootstrap::getErrorLevelString($error['type']), + $expectedMessage, + $expectedContext + )->shouldBeCalledTimes(1); + Bootstrap::$psrBatchLogger = $this->psrBatchLogger->reveal(); + MockValues::$errorReporting = $error['type']; // always match + BootStrap::errorHandler( + $error['type'], + $error['message'], + $error['file'], + $error['line'] + ); + } + + public function testErrorHandlerWithMinorError() + { + Bootstrap::$psrBatchLogger = null; + MockValues::$errorReporting = 0; + $result = BootStrap::errorHandler( + E_ERROR, + 'message', + 'file', + 1 + ); + $this->assertTrue($result); + } + + public function testErrorHandlerWithoutLogger() { + Bootstrap::$psrBatchLogger = null; + MockValues::$errorReporting = E_ERROR; + $result = BootStrap::errorHandler( + E_ERROR, + 'message', + 'file', + 1 + ); + $this->assertFalse($result); + } + + /** + * @dataProvider errorsAndMetadataProvider + */ + public function testShutdownHandler( + $error, + $resource, + $projectId, + $serviceId, + $versionId + ) { + $metadataProvider = new SimpleMetadataProvider( + $resource, + $projectId, + $serviceId, + $versionId + ); + MockValues::$type = $error['type']; + MockValues::$message = $error['message']; + MockValues::$file = $error['file']; + MockValues::$line = $error['line']; + + $fatalErrors = [E_ERROR, E_PARSE, E_COMPILE_ERROR, E_COMPILE_WARNING, + E_CORE_ERROR, E_CORE_WARNING]; + if (!in_array($error['type'], $fatalErrors, true)) { + // The shutdownHandler should not do anything, so it should pass + // with the empty psrBatchLogger mock. + Bootstrap::$psrBatchLogger = $this->psrBatchLogger->reveal(); + $this->assertNull(BootStrap::shutdownHandler()); + return; + } + $this->psrBatchLogger->getMetadataProvider() + ->willReturn($metadataProvider) + ->shouldBeCalledTimes(2); + $expectedMessage = sprintf( + '%s: %s in %s on line %d', + Bootstrap::getErrorPrefix($error['type']), + $error['message'], + $error['file'], + $error['line'] + ); + $expectedContext = [ + 'context' => [ + 'reportLocation' => [ + 'filePath' => $error['file'], + 'lineNumber' => $error['line'], + 'functionName' => 'unknown' + ] + ], + 'serviceContext' => [ + 'service' => $serviceId, + 'version' => $versionId, + ] + ]; + $this->psrBatchLogger->log( + Bootstrap::getErrorLevelString($error['type']), + $expectedMessage, + $expectedContext + )->shouldBeCalledTimes(1); + Bootstrap::$psrBatchLogger = $this->psrBatchLogger->reveal(); + BootStrap::shutdownHandler(); + } + + public function errorsAndMetadataProvider() + { + return [ + [ + ['type' => E_ERROR, + 'message' => 'error message', + 'file' => '/app/web/index.php', + 'line' => 1], + ['type' => 'my-type'], + 'my-project', + 'my-service', + 'my-version' + ], + [ + ['type' => E_WARNING, + 'message' => 'warning message', + 'file' => '/app/web/phpinfo.php', + 'line' => 2], + ['type' => 'another-type'], + 'another-project', + 'another-service', + 'another-version' + ], + ]; + } +} diff --git a/tests/unit/ErrorReporting/fakeGlobalFunctions.php b/tests/unit/ErrorReporting/fakeGlobalFunctions.php new file mode 100644 index 000000000000..aad6a31c75da --- /dev/null +++ b/tests/unit/ErrorReporting/fakeGlobalFunctions.php @@ -0,0 +1,34 @@ + MockValues::$type, + 'message' => MockValues::$message, + 'file' => MockValues::$file, + 'line' => MockValues::$line + ]; + }; + + function error_reporting() + { + return MockValues::$errorReporting; + }; + + function fwrite($target, $message) + { + if ($target != STDERR) { + throw new \RuntimeException('Only for STDERR'); + } + MockValues::$stderr = $message; + }; +} diff --git a/tests/unit/Logging/LoggingClientTest.php b/tests/unit/Logging/LoggingClientTest.php index 552cc0d93773..72ff59a341e9 100644 --- a/tests/unit/Logging/LoggingClientTest.php +++ b/tests/unit/Logging/LoggingClientTest.php @@ -45,6 +45,27 @@ public function setUp() $this->client = new LoggingTestClient(['projectId' => $this->projectId]); } + public function testPsrBatchLogger() + { + $psrBatchLogger = LoggingClient::psrBatchLogger('app'); + $this->assertInstanceOf(PsrLogger::class, $psrBatchLogger); + $r = new \ReflectionObject($psrBatchLogger); + $p = $r->getProperty('batchEnabled'); + $p->setAccessible(true); + $this->assertTrue($p->getValue($psrBatchLogger)); + $psrBatchLogger = LoggingClient::psrBatchLogger( + 'app', + ['clientConfig' => ['projectId' => 'my-project']]); + $this->assertInstanceOf(PsrLogger::class, $psrBatchLogger); + $r = new \ReflectionObject($psrBatchLogger); + $p = $r->getProperty('clientConfig'); + $p->setAccessible(true); + $this->assertEquals( + ['projectId' => 'my-project'], + $p->getValue($psrBatchLogger) + ); + } + public function testCreatesSink() { $destination = 'storage.googleapis.com/my-bucket'; diff --git a/tests/unit/Logging/PsrLoggerBatchTest.php b/tests/unit/Logging/PsrLoggerBatchTest.php new file mode 100644 index 000000000000..9b302a5d8860 --- /dev/null +++ b/tests/unit/Logging/PsrLoggerBatchTest.php @@ -0,0 +1,232 @@ +runner = $this->prophesize(BatchRunner::class); + $this->logger = $this->prophesize(Logger::class); + } + + /** + * @dataProvider optionProvider + */ + public function testSend( + $logName, + $options, + $expectedOutput + ) { + $this->logger->writeBatch(Argument::any()) + ->willReturn(true) + ->shouldBeCalledTimes(1); + $this->logger->name() + ->willReturn($logName) + ->shouldBeCalledTimes(2); + $logger = $this->logger->reveal(); + $temp = fopen('php://temp', 'rw'); + $psrBatchLogger = new PsrLogger( + $logger, + null, + $options + ['debugOutputResource' => $temp] + ); + $class = + new \ReflectionClass('\\Google\\Cloud\\Logging\\PsrLogger'); + $prop = $class->getProperty('loggers'); + $prop->setAccessible(true); + $prop = $prop->setValue([$logName => $logger]); + $psrBatchLogger->send([new Entry()]); + rewind($temp); + $output = stream_get_contents($temp); + + if ($expectedOutput === false) { + $this->assertEmpty($output); + } else { + $this->assertContains($expectedOutput, $output); + } + } + + /** + * @dataProvider traceIdProvider + */ + public function testTraceIdLabelOnGAEFlex( + $traceId, + $labels, + $expectedLabels + ) { + if (empty($traceId)) { + $server = []; + unset($_SERVER['HTTP_X_CLOUD_TRACE_CONTEXT']); + } else { + $server = ['HTTP_X_CLOUD_TRACE_CONTEXT' => $traceId]; + } + $this->runner->submitItem( + 'stackdriver-logging-my-log', Argument::any() + ) + ->will(function($args) { + self::$logName = $args[0]; + self::$entry = $args[1]; + }) + ->shouldBeCalledTimes(1); + $this->runner->registerJob( + Argument::any(), Argument::any(), Argument::any() + )->willReturn(true); + $logger = new Logger( + $this->prophesize(Rest::class)->reveal(), + self::LOG_NAME, + 'my-project' + ); + $psrBatchLogger = new PsrLogger( + $logger, + null, + [ + 'batchEnabled' => true, + 'batchRunner' => $this->runner->reveal(), + 'metadataProvider' => new GaeFlexMetadataProvider($server) + ] + ); + $psrBatchLogger->info( + 'test log', + ['stackdriverOptions' => ['labels' => $labels]] + ); + $this->assertEquals('stackdriver-logging-my-log', self::$logName); + $info = self::$entry->info(); + $this->assertEquals($expectedLabels, $info['labels']); + } + + /** + * @dataProvider levelProvider + */ + public function testWritesEntryWithLevels($level) + { + $this->runner->submitItem( + 'stackdriver-logging-my-log', Argument::any() + ) + ->will(function($args) { + self::$logName = $args[0]; + self::$entry = $args[1]; + }) + ->shouldBeCalledTimes(1); + $this->runner->registerJob( + Argument::any(), Argument::any(), Argument::any() + )->willReturn(true); + $logger = new Logger( + $this->prophesize(Rest::class)->reveal(), + self::LOG_NAME, + 'my-project' + ); + $psrBatchLogger = new PsrLogger( + $logger, + null, + [ + 'batchEnabled' => true, + 'batchRunner' => $this->runner->reveal() + ] + ); + $psrBatchLogger->$level('test log'); + $this->assertEquals('stackdriver-logging-my-log', self::$logName); + $info = self::$entry->info(); + $this->assertEquals( + array_flip(Logger::getLogLevelMap())[$level], + $info['severity'] + ); + } + + public function traceIdProvider() + { + return [ + [ + '', + [], + [], + ], + [ + str_repeat('x', 32), + [], + ['appengine.googleapis.com/trace_id' => str_repeat('x', 32)] + ], + [ + str_repeat('x', 32), + ['myKey' => 'myVal'], + [ + 'appengine.googleapis.com/trace_id' => str_repeat('x', 32), + 'myKey' => 'myVal' + ] + ], + ]; + } + + public function optionProvider() + { + return [ + [ + 'log1', + [ + 'batchEnabled' => true, + 'debugOutput' => true + ], + 'seconds for writeBatch', + ], + [ + 'log2', + [ + 'batchEnabled' => true, + 'debugOutput' => false + ], + false, + ], + ]; + } + + public function levelProvider() + { + return [ + ['EMERGENCY'], + ['ALERT'], + ['CRITICAL'], + ['ERROR'], + ['WARNING'], + ['NOTICE'], + ['INFO'], + ['DEBUG'] + ]; + } + +} diff --git a/tests/unit/Logging/PsrLoggerCompatabilityTest.php b/tests/unit/Logging/PsrLoggerCompatibilityTest.php similarity index 96% rename from tests/unit/Logging/PsrLoggerCompatabilityTest.php rename to tests/unit/Logging/PsrLoggerCompatibilityTest.php index f410bbc6318d..6f1f768862b9 100644 --- a/tests/unit/Logging/PsrLoggerCompatabilityTest.php +++ b/tests/unit/Logging/PsrLoggerCompatibilityTest.php @@ -26,7 +26,7 @@ /** * @group logging */ -class PsrLoggerCompatabilityTest extends LoggerInterfaceTest +class PsrLoggerCompatibilityTest extends LoggerInterfaceTest { public static $logs = []; diff --git a/tests/unit/Logging/PsrLoggerTest.php b/tests/unit/Logging/PsrLoggerTest.php index d1fb6c574447..3037e5bf5938 100644 --- a/tests/unit/Logging/PsrLoggerTest.php +++ b/tests/unit/Logging/PsrLoggerTest.php @@ -17,6 +17,7 @@ namespace Google\Cloud\Tests\Unit\Logging; +use Google\Cloud\Core\Report\EmptyMetadataProvider; use Google\Cloud\Logging\Logger; use Google\Cloud\Logging\PsrLogger; use Google\Cloud\Logging\Connection\ConnectionInterface; @@ -44,7 +45,7 @@ public function setUp() public function getPsrLogger($connection, array $resource = null, array $labels = null, $messageKey = 'message') { $logger = new Logger($connection->reveal(), $this->logName, $this->projectId, $resource, $labels); - return new PsrLogger($logger, $messageKey); + return new PsrLogger($logger, $messageKey, ['metadataProvider' => new EmptyMetadataProvider()]); } /** diff --git a/tests/unit/PubSub/BatchPublisherTest.php b/tests/unit/PubSub/BatchPublisherTest.php new file mode 100644 index 000000000000..daa99573d9c6 --- /dev/null +++ b/tests/unit/PubSub/BatchPublisherTest.php @@ -0,0 +1,70 @@ + 'Hello, world!']; + $runner = $this->prophesize(BatchRunner::class); + $runner->submitItem('pubsub-topic-' . self::TOPIC_NAME, $message) + ->willReturn(true) + ->shouldBeCalledTimes(1); + $runner->registerJob( + Argument::type('string'), + Argument::type('array'), + Argument::type('array') + ) + ->willReturn(true) + ->shouldBeCalledTimes(1); + + $publisher = new BatchPublisher(self::TOPIC_NAME, [ + 'batchRunner' => $runner->reveal() + ]); + + $publisher->publish($message); + } + + public function testGetCallback() + { + $callbackArray = (new TestBatchPublisher(self::TOPIC_NAME)) + ->getCallbackArray(); + + $this->assertInstanceOf(Topic::class, $callbackArray[0]); + $this->assertEquals('publishBatch', $callbackArray[1]); + } +} + +class TestBatchPublisher extends BatchPublisher +{ + public function getCallbackArray() + { + return $this->getCallback(); + } +} diff --git a/tests/unit/PubSub/TopicTest.php b/tests/unit/PubSub/TopicTest.php index eb4da74063b4..694fb3dd0967 100644 --- a/tests/unit/PubSub/TopicTest.php +++ b/tests/unit/PubSub/TopicTest.php @@ -20,6 +20,7 @@ use Google\Cloud\Core\Exception\NotFoundException; use Google\Cloud\Core\Iam\Iam; use Google\Cloud\Core\Iterator\ItemIterator; +use Google\Cloud\PubSub\BatchPublisher; use Google\Cloud\PubSub\Connection\ConnectionInterface; use Google\Cloud\PubSub\Subscription; use Google\Cloud\PubSub\Topic; @@ -238,6 +239,14 @@ public function testPublishMalformedMessage() $this->topic->publishBatch([$message]); } + public function testBatchPublisher() + { + $this->assertInstanceOf( + BatchPublisher::class, + $this->topic->batchPublisher() + ); + } + public function testSubscribe() { $subscriptionData = [ diff --git a/tests/unit/Trace/Reporter/AsyncReporterTest.php b/tests/unit/Trace/Reporter/AsyncReporterTest.php new file mode 100644 index 000000000000..c30cb8d936fc --- /dev/null +++ b/tests/unit/Trace/Reporter/AsyncReporterTest.php @@ -0,0 +1,102 @@ +runner = $this->prophesize(BatchRunner::class); + $this->tracer = $this->prophesize(TracerInterface::class); + + $this->runner->registerJob( + Argument::type('string'), + Argument::type('array'), + Argument::type('array') + )->willReturn(true); + } + + public function testReportsTrace() + { + $spans = [ + new TraceSpan([ + 'name' => 'span', + 'startTime' => microtime(true), + 'endTime' => microtime(true) + 10 + ]) + ]; + $traceId = 'testtraceid'; + $this->tracer->context()->willReturn(new TraceContext($traceId)); + $this->tracer->spans()->willReturn($spans); + $this->runner->submitItem(Argument::type('string'), Argument::type(Trace::class)) + ->willReturn(true); + $trace = $this->prophesize(Trace::class); + $trace->setSpans(Argument::any())->shouldBeCalled(); + $client = $this->prophesize(TraceClient::class); + $client->insertBatch([$trace])->willReturn(true); + $client->trace($traceId)->willReturn($trace)->shouldBeCalled(); + + $reporter = new AsyncReporter([ + 'client' => $client->reveal(), + 'batchRunner' => $this->runner->reveal() + ]); + $this->assertTrue($reporter->report($this->tracer->reveal())); + } + + public function testSkipsReportingWhenNoSpans() + { + $this->tracer->spans()->willReturn([]); + + $reporter = new AsyncReporter([ + 'batchRunner' => $this->runner->reveal() + ]); + $this->assertFalse($reporter->report($this->tracer->reveal())); + } + + public function testGetCallback() + { + $callbackArray = (new TestAsyncReporter()) + ->getCallbackArray(); + + $this->assertInstanceOf(TraceClient::class, $callbackArray[0]); + $this->assertEquals('insertBatch', $callbackArray[1]); + } +} + +class TestAsyncReporter extends AsyncReporter +{ + public function getCallbackArray() + { + return $this->getCallback(); + } +}