diff options
Diffstat (limited to 'pkgs/development/python-modules/tensorflow')
7 files changed, 701 insertions, 34 deletions
diff --git a/pkgs/development/python-modules/tensorflow/absl_py_argparse_flags.patch b/pkgs/development/python-modules/tensorflow/absl_py_argparse_flags.patch new file mode 100644 index 00000000000..1b130e22662 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/absl_py_argparse_flags.patch @@ -0,0 +1,11 @@ +diff -Naurd x/third_party/systemlibs/absl_py.absl.flags.BUILD y/third_party/systemlibs/absl_py.absl.flags.BUILD +--- x/third_party/systemlibs/absl_py.absl.flags.BUILD 2023-09-20 06:02:15.079683729 +0000 ++++ y/third_party/systemlibs/absl_py.absl.flags.BUILD 2023-09-20 06:04:58.491897303 +0000 +@@ -5,3 +5,7 @@ + py_library( + name = "flags", + ) ++ ++py_library( ++ name = "argparse_flags", ++) diff --git a/pkgs/development/python-modules/tensorflow/absl_to_std.patch b/pkgs/development/python-modules/tensorflow/absl_to_std.patch new file mode 100644 index 00000000000..b834ec983c9 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/absl_to_std.patch @@ -0,0 +1,389 @@ +diff -Naurd x/tensorflow/c/experimental/stream_executor/stream_executor.cc y/tensorflow/c/experimental/stream_executor/stream_executor.cc +--- x/tensorflow/c/experimental/stream_executor/stream_executor.cc 2023-10-19 14:25:48.648646416 +0000 ++++ y/tensorflow/c/experimental/stream_executor/stream_executor.cc 2023-10-19 14:29:11.700743574 +0000 +@@ -23,6 +23,7 @@ + + #include <string> + #include <utility> ++#include <optional> + + #include "absl/functional/any_invocable.h" + #include "tensorflow/c/c_api_macros.h" +@@ -275,17 +276,17 @@ + stream_executor_->unified_memory_deallocate(&device_, mem); + } + +- absl::optional<AllocatorStats> GetAllocatorStats() override { ++ std::optional<AllocatorStats> GetAllocatorStats() override { + SP_AllocatorStats c_stats{SP_ALLOCATORSTATS_STRUCT_SIZE}; + TF_Bool has_stats = + stream_executor_->get_allocator_stats(&device_, &c_stats); + if (!has_stats) { +- return absl::nullopt; ++ return std::nullopt; + } + tsl::Status status = ValidateSPAllocatorStats(c_stats); + if (!status.ok()) { + LOG(ERROR) << status.message(); +- return absl::nullopt; ++ return std::nullopt; + } + ::stream_executor::AllocatorStats stats; + stats.num_allocs = c_stats.num_allocs; +diff -Naurd x/tensorflow/c/experimental/stream_executor/stream_executor_test.cc y/tensorflow/c/experimental/stream_executor/stream_executor_test.cc +--- x/tensorflow/c/experimental/stream_executor/stream_executor_test.cc 2023-10-19 14:25:48.648646416 +0000 ++++ y/tensorflow/c/experimental/stream_executor/stream_executor_test.cc 2023-10-19 14:29:11.700743574 +0000 +@@ -15,6 +15,7 @@ + #include "tensorflow/c/experimental/stream_executor/stream_executor.h" + + #include <utility> ++#include <optional> + + #include "tensorflow/c/experimental/stream_executor/stream_executor_internal.h" + #include "tensorflow/c/experimental/stream_executor/stream_executor_test_util.h" +@@ -239,7 +240,7 @@ + }; + + StreamExecutor* executor = GetExecutor(0); +- absl::optional<AllocatorStats> optional_stats = executor->GetAllocatorStats(); ++ std::optional<AllocatorStats> optional_stats = executor->GetAllocatorStats(); + ASSERT_TRUE(optional_stats.has_value()); + AllocatorStats stats = optional_stats.value(); + ASSERT_EQ(stats.bytes_in_use, 123); +diff -Naurd x/tensorflow/compiler/xla/stream_executor/allocator_stats.h y/tensorflow/compiler/xla/stream_executor/allocator_stats.h +--- x/tensorflow/compiler/xla/stream_executor/allocator_stats.h 2023-10-19 14:25:55.064649379 +0000 ++++ y/tensorflow/compiler/xla/stream_executor/allocator_stats.h 2023-10-19 14:29:11.700743574 +0000 +@@ -17,6 +17,7 @@ + #define TENSORFLOW_COMPILER_XLA_STREAM_EXECUTOR_ALLOCATOR_STATS_H_ + + #include <string> ++#include <optional> + + #include "absl/types/optional.h" + #include "tensorflow/compiler/xla/stream_executor/platform/port.h" +diff -Naurd x/tensorflow/core/common_runtime/mkl_cpu_allocator.h y/tensorflow/core/common_runtime/mkl_cpu_allocator.h +--- x/tensorflow/core/common_runtime/mkl_cpu_allocator.h 2023-10-19 14:25:58.996651199 +0000 ++++ y/tensorflow/core/common_runtime/mkl_cpu_allocator.h 2023-10-19 14:29:11.700743574 +0000 +@@ -22,6 +22,7 @@ + #ifdef INTEL_MKL + + #include <cstdlib> ++#include <optional> + + #include "tensorflow/core/common_runtime/bfc_allocator.h" + #include "tensorflow/core/common_runtime/pool_allocator.h" +@@ -80,7 +81,7 @@ + port::AlignedFree(ptr); + } + +- absl::optional<AllocatorStats> GetStats() override { ++ std::optional<AllocatorStats> GetStats() override { + mutex_lock l(mutex_); + return stats_; + } +@@ -242,7 +243,7 @@ + large_size_allocator_->DeallocateRaw(ptr); + } + } +- absl::optional<AllocatorStats> GetStats() override { ++ std::optional<AllocatorStats> GetStats() override { + auto s_stats = small_size_allocator_->GetStats(); + auto l_stats = large_size_allocator_->GetStats(); + +diff -Naurd x/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.cc y/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.cc +--- x/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.cc 2023-10-19 14:25:59.236651310 +0000 ++++ y/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.cc 2023-10-19 14:29:11.700743574 +0000 +@@ -32,7 +32,7 @@ + return sub_allocator_->Free(ptr, 0); + } + +-absl::optional<AllocatorStats> PluggableDeviceSimpleAllocator::GetStats() { ++std::optional<AllocatorStats> PluggableDeviceSimpleAllocator::GetStats() { + AllocatorStats stats_; + stats_.num_allocs = 0; + stats_.peak_bytes_in_use = 0; +diff -Naurd x/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.h y/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.h +--- x/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.h 2023-10-19 14:25:59.236651310 +0000 ++++ y/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.h 2023-10-19 14:29:11.700743574 +0000 +@@ -19,6 +19,7 @@ + #include <string> + #include <unordered_map> + #include <vector> ++#include <optional> + + #include "tensorflow/core/common_runtime/device/device_mem_allocator.h" + #include "tensorflow/core/platform/thread_annotations.h" +@@ -37,7 +38,7 @@ + + bool TracksAllocationSizes() const override { return false; } + string Name() override { return "Simple allocator"; } +- absl::optional<AllocatorStats> GetStats() override; ++ std::optional<AllocatorStats> GetStats() override; + + AllocatorMemoryType GetMemoryType() const override { + return sub_allocator_->GetMemoryType(); +diff -Naurd x/tensorflow/core/common_runtime/process_state.h y/tensorflow/core/common_runtime/process_state.h +--- x/tensorflow/core/common_runtime/process_state.h 2023-10-19 14:25:59.076651236 +0000 ++++ y/tensorflow/core/common_runtime/process_state.h 2023-10-19 14:29:11.704743576 +0000 +@@ -20,6 +20,7 @@ + #include <map> + #include <unordered_map> + #include <vector> ++#include <optional> + + #include "tensorflow/core/framework/allocator.h" + #include "tensorflow/core/framework/allocator_registry.h" +@@ -144,7 +145,7 @@ + size_t AllocatedSize(const void* p) const override { + return a_->AllocatedSize(p); + } +- absl::optional<AllocatorStats> GetStats() override { return a_->GetStats(); } ++ std::optional<AllocatorStats> GetStats() override { return a_->GetStats(); } + bool ClearStats() override { return a_->ClearStats(); } + + AllocatorMemoryType GetMemoryType() const override { +diff -Naurd x/tensorflow/core/common_runtime/step_stats_collector.cc y/tensorflow/core/common_runtime/step_stats_collector.cc +--- x/tensorflow/core/common_runtime/step_stats_collector.cc 2023-10-19 14:25:59.112651253 +0000 ++++ y/tensorflow/core/common_runtime/step_stats_collector.cc 2023-10-19 14:29:11.704743576 +0000 +@@ -15,6 +15,7 @@ + #include "tensorflow/core/common_runtime/step_stats_collector.h" + + #include <memory> ++#include <optional> + + #include "tensorflow/core/common_runtime/costmodel_manager.h" + #include "tensorflow/core/framework/allocation_description.pb.h" +@@ -175,7 +176,7 @@ + memory->set_peak_bytes(std::get<1>(sizes)); + memory->set_live_bytes(std::get<2>(sizes)); + +- absl::optional<AllocatorStats> stats = allocator->GetStats(); ++ std::optional<AllocatorStats> stats = allocator->GetStats(); + if (stats) { + memory->set_allocator_bytes_in_use(stats->bytes_in_use); + } +diff -Naurd x/tensorflow/core/framework/allocator_test.cc y/tensorflow/core/framework/allocator_test.cc +--- x/tensorflow/core/framework/allocator_test.cc 2023-10-19 14:25:59.524651443 +0000 ++++ y/tensorflow/core/framework/allocator_test.cc 2023-10-19 14:29:11.704743576 +0000 +@@ -17,6 +17,7 @@ + + #include <algorithm> + #include <vector> ++#include <optional> + + #include "tensorflow/core/framework/typed_allocator.h" + #include "tensorflow/core/platform/logging.h" +@@ -33,7 +34,7 @@ + + static void CheckStats(Allocator* a, int64_t num_allocs, int64_t bytes_in_use, + int64_t peak_bytes_in_use, int64_t largest_alloc_size) { +- absl::optional<AllocatorStats> stats = a->GetStats(); ++ std::optional<AllocatorStats> stats = a->GetStats(); + EXPECT_TRUE(stats); + if (!stats) { + return; +@@ -255,7 +256,7 @@ + EXPECT_EQ(e0.Name(), "MemoryAllocation") + << "XSpace: " << xspace.DebugString(); + { +- absl::optional<std::string> bytes_allocated, peak_bytes_in_use, ++ std::optional<std::string> bytes_allocated, peak_bytes_in_use, + requested_bytes, allocation_bytes; + e0.ForEachStat([&](const ::tensorflow::profiler::XStatVisitor& stat) { + LOG(ERROR) << "STAT " << stat.Name() << ": " << stat.ToString(); +@@ -282,7 +283,7 @@ + EXPECT_EQ(e1.Name(), "MemoryDeallocation") + << "XSpace: " << xspace.DebugString(); + { +- absl::optional<std::string> bytes_allocated, peak_bytes_in_use, ++ std::optional<std::string> bytes_allocated, peak_bytes_in_use, + allocation_bytes; + e1.ForEachStat([&](const ::tensorflow::profiler::XStatVisitor& stat) { + if (stat.Name() == "bytes_allocated") { +diff -Naurd x/tensorflow/core/framework/tracking_allocator_test.cc y/tensorflow/core/framework/tracking_allocator_test.cc +--- x/tensorflow/core/framework/tracking_allocator_test.cc 2023-10-19 14:25:59.700651525 +0000 ++++ y/tensorflow/core/framework/tracking_allocator_test.cc 2023-10-19 14:29:11.704743576 +0000 +@@ -16,6 +16,7 @@ + #include "tensorflow/core/framework/tracking_allocator.h" + + #include <unordered_map> ++#include <optional> + + #include "tensorflow/core/framework/allocator.h" + #include "tensorflow/core/platform/logging.h" +@@ -44,7 +45,7 @@ + EXPECT_NE(size_map_.end(), iter); + return iter->second; + } +- absl::optional<AllocatorStats> GetStats() override { return absl::nullopt; } ++ std::optional<AllocatorStats> GetStats() override { return std::nullopt; } + + private: + std::unordered_map<const void*, size_t> size_map_; +@@ -58,7 +59,7 @@ + } + void DeallocateRaw(void* ptr) override {} + bool TracksAllocationSizes() const override { return true; } +- absl::optional<AllocatorStats> GetStats() override { return absl::nullopt; } ++ std::optional<AllocatorStats> GetStats() override { return std::nullopt; } + }; + + TEST(TrackingAllocatorTest, SimpleNoTracking) { +diff -Naurd x/tensorflow/core/grappler/clusters/single_machine.cc y/tensorflow/core/grappler/clusters/single_machine.cc +--- x/tensorflow/core/grappler/clusters/single_machine.cc 2023-10-19 14:25:59.964651648 +0000 ++++ y/tensorflow/core/grappler/clusters/single_machine.cc 2023-10-19 14:29:11.704743576 +0000 +@@ -17,6 +17,7 @@ + + #include <atomic> + #include <memory> ++#include <optional> + + #include "tensorflow/cc/training/queue_runner.h" + #include "tensorflow/core/common_runtime/device.h" +@@ -230,7 +231,7 @@ + return Status(absl::StatusCode::kInvalidArgument, + "Tracking allocation is not enabled."); + } +- absl::optional<AllocatorStats> stats = allocator->GetStats(); ++ std::optional<AllocatorStats> stats = allocator->GetStats(); + (*device_peak_memory)[device->name()] = + (stats ? stats->peak_bytes_in_use : 0); + } +diff -Naurd x/tensorflow/core/kernels/stack.cc y/tensorflow/core/kernels/stack.cc +--- x/tensorflow/core/kernels/stack.cc 2023-10-19 14:26:01.668652437 +0000 ++++ y/tensorflow/core/kernels/stack.cc 2023-10-19 14:29:11.704743576 +0000 +@@ -18,6 +18,7 @@ + #include <limits.h> + #include <atomic> + #include <vector> ++#include <optional> + + #include "tensorflow/core/common_runtime/device.h" + #include "tensorflow/core/framework/device_base.h" +@@ -245,7 +246,7 @@ + DeviceContext* device_ctxt = ctx->op_device_context(); + auto device = static_cast<tensorflow::Device*>(ctx->device()); + Allocator* allocator = device->GetAllocator(alloc_attrs); +- absl::optional<AllocatorStats> stats = allocator->GetStats(); ++ std::optional<AllocatorStats> stats = allocator->GetStats(); + if (stats && *stats->bytes_limit && + stats->bytes_in_use > (*stats->bytes_limit * kOccupancy)) { + // Asynchronously copy the tensor from GPU to CPU memory. +diff -Naurd x/tensorflow/python/tfe_wrapper.cc y/tensorflow/python/tfe_wrapper.cc +--- x/tensorflow/python/tfe_wrapper.cc 2023-10-19 14:26:10.716656639 +0000 ++++ y/tensorflow/python/tfe_wrapper.cc 2023-10-19 14:29:11.708743578 +0000 +@@ -14,6 +14,7 @@ + ==============================================================================*/ + + #include <memory> ++#include <optional> + + #include "Python.h" + #include "absl/strings/match.h" +@@ -691,7 +692,7 @@ + tensorflow::AllocatorAttributes attrs; + tensorflow::Allocator* allocator = matched_device->GetAllocator(attrs); + +- if (absl::optional<tensorflow::AllocatorStats> stats = ++ if (std::optional<tensorflow::AllocatorStats> stats = + allocator->GetStats()) { + return std::map<std::string, int64_t>{{"current", stats->bytes_in_use}, + {"peak", stats->peak_bytes_in_use}}; +diff -Naurd x/tensorflow/tsl/framework/allocator.h y/tensorflow/tsl/framework/allocator.h +--- x/tensorflow/tsl/framework/allocator.h 2023-10-19 14:26:15.884659044 +0000 ++++ y/tensorflow/tsl/framework/allocator.h 2023-10-19 14:29:11.708743578 +0000 +@@ -216,7 +216,7 @@ + } + + // Fills in 'stats' with statistics collected by this allocator. +- virtual absl::optional<AllocatorStats> GetStats() { return absl::nullopt; } ++ virtual std::optional<AllocatorStats> GetStats() { return std::nullopt; } + + // If implemented, clears the internal stats except for the `in_use` fields + // and sets the `peak_bytes_in_use` to be equal to the `bytes_in_use`. Returns +diff -Naurd x/tensorflow/tsl/framework/bfc_allocator.cc y/tensorflow/tsl/framework/bfc_allocator.cc +--- x/tensorflow/tsl/framework/bfc_allocator.cc 2023-10-19 14:26:15.900659052 +0000 ++++ y/tensorflow/tsl/framework/bfc_allocator.cc 2023-10-19 14:29:11.708743578 +0000 +@@ -1205,7 +1205,7 @@ + return md; + } + +-absl::optional<AllocatorStats> BFCAllocator::GetStats() { ++std::optional<AllocatorStats> BFCAllocator::GetStats() { + mutex_lock l(lock_); + return stats_; + } +diff -Naurd x/tensorflow/tsl/framework/bfc_allocator.h y/tensorflow/tsl/framework/bfc_allocator.h +--- x/tensorflow/tsl/framework/bfc_allocator.h 2023-10-19 14:26:15.900659052 +0000 ++++ y/tensorflow/tsl/framework/bfc_allocator.h 2023-10-19 14:29:11.708743578 +0000 +@@ -22,6 +22,7 @@ + #include <string> + #include <unordered_map> + #include <vector> ++#include <optional> + + #include "absl/container/flat_hash_set.h" + #include "tensorflow/tsl/framework/allocator.h" +@@ -93,7 +94,7 @@ + + int64_t AllocationId(const void* ptr) const override; + +- absl::optional<AllocatorStats> GetStats() override; ++ std::optional<AllocatorStats> GetStats() override; + + bool ClearStats() override; + +diff -Naurd x/tensorflow/tsl/framework/cpu_allocator_impl.cc y/tensorflow/tsl/framework/cpu_allocator_impl.cc +--- x/tensorflow/tsl/framework/cpu_allocator_impl.cc 2023-10-19 14:26:15.928659065 +0000 ++++ y/tensorflow/tsl/framework/cpu_allocator_impl.cc 2023-10-19 14:29:11.708743578 +0000 +@@ -15,6 +15,7 @@ + + #include <algorithm> + #include <atomic> ++#include <optional> + + #include "tensorflow/tsl/framework/allocator.h" + #include "tensorflow/tsl/framework/allocator_registry.h" +@@ -145,8 +146,8 @@ + /*level=*/tsl::profiler::TraceMeLevel::kInfo); + } + +- absl::optional<AllocatorStats> GetStats() override { +- if (!cpu_allocator_collect_stats) return absl::nullopt; ++ std::optional<AllocatorStats> GetStats() override { ++ if (!cpu_allocator_collect_stats) return std::nullopt; + mutex_lock l(mu_); + return stats_; + } +diff -Naurd x/tensorflow/tsl/framework/tracking_allocator.cc y/tensorflow/tsl/framework/tracking_allocator.cc +--- x/tensorflow/tsl/framework/tracking_allocator.cc 2023-10-19 14:26:15.968659084 +0000 ++++ y/tensorflow/tsl/framework/tracking_allocator.cc 2023-10-19 14:29:11.708743578 +0000 +@@ -152,7 +152,7 @@ + } + } + +-absl::optional<AllocatorStats> TrackingAllocator::GetStats() { ++std::optional<AllocatorStats> TrackingAllocator::GetStats() { + return allocator_->GetStats(); + } + +diff -Naurd x/tensorflow/tsl/framework/tracking_allocator.h y/tensorflow/tsl/framework/tracking_allocator.h +--- x/tensorflow/tsl/framework/tracking_allocator.h 2023-10-19 14:26:15.968659084 +0000 ++++ y/tensorflow/tsl/framework/tracking_allocator.h 2023-10-19 14:29:11.712743580 +0000 +@@ -17,6 +17,7 @@ + #define TENSORFLOW_TSL_FRAMEWORK_TRACKING_ALLOCATOR_H_ + + #include <unordered_map> ++#include <optional> + + #include "tensorflow/tsl/framework/allocator.h" + #include "tensorflow/tsl/lib/gtl/inlined_vector.h" +@@ -66,7 +67,7 @@ + size_t RequestedSize(const void* ptr) const override; + size_t AllocatedSize(const void* ptr) const override; + int64_t AllocationId(const void* ptr) const override; +- absl::optional<AllocatorStats> GetStats() override; ++ std::optional<AllocatorStats> GetStats() override; + bool ClearStats() override; + + AllocatorMemoryType GetMemoryType() const override { diff --git a/pkgs/development/python-modules/tensorflow/com_google_absl_add_log.patch b/pkgs/development/python-modules/tensorflow/com_google_absl_add_log.patch new file mode 100644 index 00000000000..ebb2fb129c0 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/com_google_absl_add_log.patch @@ -0,0 +1,166 @@ +diff -Naurd x/third_party/absl/system.absl.base.BUILD y/third_party/absl/system.absl.base.BUILD +--- x/third_party/absl/system.absl.base.BUILD 2023-09-17 09:12:05.499753364 +0000 ++++ y/third_party/absl/system.absl.base.BUILD 2023-09-17 09:16:01.200082822 +0000 +@@ -22,7 +22,12 @@ + + cc_library( + name = "raw_logging_internal", +- linkopts = ["-labsl_raw_logging_internal"], ++ linkopts = [ ++ "-labsl_raw_logging_internal", ++ "-labsl_log_internal_conditions", ++ "-labsl_log_internal_message", ++ "-labsl_log_internal_nullguard", ++ ], + visibility = [ + "//absl:__subpackages__", + ], +diff -Naurd x/third_party/absl/system.absl.log.BUILD y/third_party/absl/system.absl.log.BUILD +--- x/third_party/absl/system.absl.log.BUILD 1970-01-01 00:00:00.000000000 +0000 ++++ y/third_party/absl/system.absl.log.BUILD 2023-09-17 09:12:11.795762177 +0000 +@@ -0,0 +1,134 @@ ++load("@rules_cc//cc:defs.bzl", "cc_library") ++ ++package(default_visibility = ["//visibility:public"]) ++ ++cc_library( ++ name = "absl_check", ++ deps = [ ++ ], ++) ++ ++cc_library( ++ name = "absl_log", ++ deps = [ ++ ], ++) ++ ++cc_library( ++ name = "check", ++ deps = [ ++ ], ++) ++ ++cc_library( ++ name = "die_if_null", ++ deps = [ ++ ":log", ++ "//absl/base:config", ++ "//absl/base:core_headers", ++ "//absl/strings", ++ ], ++) ++ ++cc_library( ++ name = "flags", ++ deps = [ ++ ":globals", ++ "//absl/base:config", ++ "//absl/base:core_headers", ++ "//absl/base:log_severity", ++ "//absl/flags:flag", ++ "//absl/flags:marshalling", ++ "//absl/strings", ++ ], ++) ++ ++cc_library( ++ name = "globals", ++ deps = [ ++ "//absl/base:atomic_hook", ++ "//absl/base:config", ++ "//absl/base:core_headers", ++ "//absl/base:log_severity", ++ "//absl/hash", ++ "//absl/strings", ++ ], ++) ++ ++cc_library( ++ name = "initialize", ++ deps = [ ++ ":globals", ++ "//absl/base:config", ++ "//absl/time", ++ ], ++) ++ ++cc_library( ++ name = "log", ++ deps = [ ++ ], ++) ++ ++cc_library( ++ name = "log_entry", ++ deps = [ ++ "//absl/base:config", ++ "//absl/base:core_headers", ++ "//absl/base:log_severity", ++ "//absl/strings", ++ "//absl/time", ++ "//absl/types:span", ++ ], ++) ++ ++cc_library( ++ name = "log_sink", ++ deps = [ ++ ":log_entry", ++ "//absl/base:config", ++ ], ++) ++ ++cc_library( ++ name = "log_sink_registry", ++ deps = [ ++ ":log_sink", ++ "//absl/base:config", ++ ], ++) ++ ++cc_library( ++ name = "log_streamer", ++ deps = [ ++ ":absl_log", ++ "//absl/base:config", ++ "//absl/base:log_severity", ++ "//absl/strings", ++ "//absl/strings:internal", ++ "//absl/types:optional", ++ "//absl/utility", ++ ], ++) ++ ++cc_library( ++ name = "scoped_mock_log", ++ deps = [ ++ ":log_entry", ++ ":log_sink", ++ ":log_sink_registry", ++ "//absl/base:config", ++ "//absl/base:log_severity", ++ "//absl/base:raw_logging_internal", ++ "//absl/strings", ++ "@com_google_googletest//:gtest", ++ ], ++) ++ ++cc_library( ++ name = "structured", ++ deps = [ ++ "//absl/base:config", ++ "//absl/strings", ++ ], ++) +diff -Naurd x/third_party/absl/workspace.bzl y/third_party/absl/workspace.bzl +--- x/third_party/absl/workspace.bzl 2023-09-17 09:12:05.499753364 +0000 ++++ y/third_party/absl/workspace.bzl 2023-09-17 09:12:11.795762177 +0000 +@@ -20,6 +20,7 @@ + "flags", + "functional", + "hash", ++ "log", + "memory", + "meta", + "numeric", diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index 348f5197a9d..ff95c847eae 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -1,18 +1,18 @@ -{ stdenv, bazel_5, buildBazelPackage, isPy3k, lib, fetchFromGitHub, symlinkJoin -, addOpenGLRunpath, fetchpatch +{ stdenv, bazel_5, buildBazelPackage, lib, fetchFromGitHub, symlinkJoin +, addOpenGLRunpath, fetchpatch, fetchzip, linkFarm # Python deps , buildPythonPackage, pythonOlder, python # Python libraries -, numpy, tensorboard, absl-py -, packaging, setuptools, wheel, keras, keras-preprocessing, google-pasta +, numpy, tensorboard, abseil-cpp, absl-py +, packaging, setuptools, wheel, keras-preprocessing, google-pasta , opt-einsum, astunparse, h5py , termcolor, grpcio, six, wrapt, protobuf-python, tensorflow-estimator-bin , dill, flatbuffers-python, portpicker, tblib, typing-extensions # Common deps -, git, pybind11, which, binutils, glibcLocales, cython, perl, coreutils +, git, pybind11, which, binutils, glibcLocales, cython, perl # Common libraries , jemalloc, mpi, gast, grpc, sqlite, boringssl, jsoncpp, nsync -, curl, snappy, flatbuffers-core, lmdb-core, icu, double-conversion, libpng, libjpeg_turbo, giflib, protobuf-core +, curl, snappy, flatbuffers-core, icu, double-conversion, libpng, libjpeg_turbo, giflib, protobuf-core # Upstream by default includes cuda support since tensorflow 1.15. We could do # that in nix as well. It would make some things easier and less confusing, but # it would also make the default tensorflow package unfree. See @@ -53,7 +53,18 @@ let if cudaSupport then cudaPackages.backendStdenv else if originalStdenv.isDarwin then llvmPackages_11.stdenv else originalStdenv; - inherit (cudaPackages) cudatoolkit cudnn nccl; + inherit (cudaPackages) cudatoolkit nccl; + # use compatible cuDNN (https://www.tensorflow.org/install/source#gpu) + # cudaPackages.cudnn led to this: + # https://github.com/tensorflow/tensorflow/issues/60398 + cudnn = cudaPackages.cudnn_8_6; + gentoo-patches = fetchzip { + url = "https://dev.gentoo.org/~perfinion/patches/tensorflow-patches-2.12.0.tar.bz2"; + hash = "sha256-SCRX/5/zML7LmKEPJkcM5Tebez9vv/gmE4xhT/jyqWs="; + }; + protobuf-extra = linkFarm "protobuf-extra" [ + { name = "include"; path = protobuf-core.src; } + ]; in assert cudaSupport -> cudatoolkit != null @@ -99,7 +110,7 @@ let tfFeature = x: if x then "1" else "0"; - version = "2.11.1"; + version = "2.13.0"; variant = lib.optionalString cudaSupport "-gpu"; pname = "tensorflow${variant}"; @@ -208,14 +219,14 @@ let owner = "tensorflow"; repo = "tensorflow"; rev = "refs/tags/v${version}"; - hash = "sha256-q59cUW6613byHk4LGl+sefO5czLSWxOrSyLbJ1pkNEY="; + hash = "sha256-Rq5pAVmxlWBVnph20fkAwbfy+iuBNlfFy14poDPd5h0="; }; # On update, it can be useful to steal the changes from gentoo # https://gitweb.gentoo.org/repo/gentoo.git/tree/sci-libs/tensorflow nativeBuildInputs = [ - which pythonEnv cython perl protobuf-core + which pythonEnv cython perl protobuf-core protobuf-extra ] ++ lib.optional cudaSupport addOpenGLRunpath; buildInputs = [ @@ -225,6 +236,7 @@ let git # libs taken from system through the TF_SYS_LIBS mechanism + abseil-cpp boringssl curl double-conversion @@ -236,7 +248,6 @@ let jsoncpp libjpeg_turbo libpng - lmdb-core (pybind11.overridePythonAttrs (_: { inherit stdenv; })) snappy sqlite @@ -265,6 +276,7 @@ let "astor_archive" "astunparse_archive" "boringssl" + "com_google_absl" # Not packaged in nixpkgs # "com_github_googleapis_googleapis" # "com_github_googlecloudplatform_google_cloud_cpp" @@ -284,7 +296,6 @@ let "icu" "jsoncpp_git" "libjpeg_turbo" - "lmdb" "nasm" "opt_einsum_archive" "org_sqlite" @@ -328,6 +339,25 @@ let GCC_HOST_COMPILER_PREFIX = lib.optionalString cudaSupport "${cudatoolkit_cc_joined}/bin"; GCC_HOST_COMPILER_PATH = lib.optionalString cudaSupport "${cudatoolkit_cc_joined}/bin/cc"; + patches = [ + "${gentoo-patches}/0002-systemlib-Latest-absl-LTS-has-split-cord-libs.patch" + "${gentoo-patches}/0005-systemlib-Updates-for-Abseil-20220623-LTS.patch" + "${gentoo-patches}/0007-systemlibs-Add-well_known_types_py_pb2-target.patch" + # https://github.com/conda-forge/tensorflow-feedstock/pull/329/commits/0a63c5a962451b4da99a9948323d8b3ed462f461 + (fetchpatch { + name = "fix-layout-proto-duplicate-loading.patch"; + url = "https://raw.githubusercontent.com/conda-forge/tensorflow-feedstock/0a63c5a962451b4da99a9948323d8b3ed462f461/recipe/patches/0001-Omit-linking-to-layout_proto_cc-if-protobuf-linkage-.patch"; + hash = "sha256-/7buV6DinKnrgfqbe7KKSh9rCebeQdXv2Uj+Xg/083w="; + }) + ./com_google_absl_add_log.patch + ./absl_py_argparse_flags.patch + ./protobuf_python.patch + ./pybind11_protobuf_python_runtime_dep.patch + ./pybind11_protobuf_newer_version.patch + ] ++ lib.optionals (stdenv.hostPlatform.system == "aarch64-darwin") [ + ./absl_to_std.patch + ]; + postPatch = '' # bazel 3.3 should work just as well as bazel 3.1 rm -f .bazelversion @@ -395,18 +425,30 @@ let fetchAttrs = { sha256 = { x86_64-linux = if cudaSupport - then "sha256-lURiR0Ra4kynDXyfuONG+A7CpxnAsfKzIdFTExKzp1o=" - else "sha256-lDvRgj+UlaneRGZOO9UVCb6uyxcbRJfUhABf/sgKPi0="; - aarch64-linux = "sha256-z2d45fqHz5HW+qkv3fR9hMg3sEwUzJfxF54vng85bHk="; - x86_64-darwin = "sha256-AAvuz8o6ZRkaSYMgaep74lDDQcxOupDCX4vRaK/jnCU="; - aarch64-darwin = "sha256-kexRSvfQqb92ZRuUqAO070RnUUBidAqghiA7Y8do9vc="; + then "sha256-5VFMNHeLrUxW5RTr6EhT3pay9nWJ5JkZTGirDds5QkU=" + else "sha256-KzgWV69Btr84FdwQ5JI2nQEsqiPg1/+TWdbw5bmxXOE="; + aarch64-linux = "sha256-9btXrNHqd720oXTPDhSmFidv5iaZRLjCVX8opmrMjXk="; + x86_64-darwin = "sha256-gqb03kB0z2pZQ6m1fyRp1/Nbt8AVVHWpOJSeZNCLc4w="; + aarch64-darwin = "sha256-WdgAaFZU+ePwWkVBhLzjlNT7ELfGHOTaMdafcAMD5yo="; }.${stdenv.hostPlatform.system} or (throw "unsupported system ${stdenv.hostPlatform.system}"); }; buildAttrs = { outputs = [ "out" "python" ]; + # need to rebuild schemas since we use a different flatbuffers version preBuild = '' + (cd tensorflow/lite/schema;${flatbuffers-core}/bin/flatc --gen-object-api -c schema.fbs) + (cd tensorflow/lite/schema;${flatbuffers-core}/bin/flatc --gen-object-api -c conversion_metadata.fbs) + (cd tensorflow/lite/acceleration/configuration;${flatbuffers-core}/bin/flatc -o configuration.fbs --proto configuration.proto) + sed -i s,tflite.proto,tflite,g tensorflow/lite/acceleration/configuration/configuration.fbs/configuration.fbs + (cd tensorflow/lite/acceleration/configuration;${flatbuffers-core}/bin/flatc --gen-compare --gen-object-api -c configuration.fbs/configuration.fbs) + cp -r tensorflow/lite/acceleration/configuration/configuration.fbs tensorflow/lite/experimental/acceleration/configuration + (cd tensorflow/lite/experimental/acceleration/configuration;${flatbuffers-core}/bin/flatc -c configuration.fbs/configuration.fbs) + (cd tensorflow/lite/delegates/gpu/cl;${flatbuffers-core}/bin/flatc -c compiled_program_cache.fbs) + (cd tensorflow/lite/delegates/gpu/cl;${flatbuffers-core}/bin/flatc -I $NIX_BUILD_TOP/source -c serialization.fbs) + (cd tensorflow/lite/delegates/gpu/common;${flatbuffers-core}/bin/flatc -I $NIX_BUILD_TOP/source -c gpu_model.fbs) + (cd tensorflow/lite/delegates/gpu/common/task;${flatbuffers-core}/bin/flatc -c serialization_base.fbs) patchShebangs . ''; @@ -448,10 +490,7 @@ let license = licenses.asl20; maintainers = with maintainers; [ abbradar ]; platforms = with platforms; linux ++ darwin; - # More vulnerabilities in 2.11.1 really; https://github.com/tensorflow/tensorflow/releases - knownVulnerabilities = [ "CVE-2023-33976" ]; - broken = true || # most likely needs dealing with protobuf/abseil updates - !(xlaSupport -> cudaSupport) || python.pythonVersion == "3.11"; + broken = stdenv.isDarwin || !(xlaSupport -> cudaSupport); } // lib.optionalAttrs stdenv.isDarwin { timeout = 86400; # 24 hours maxSilent = 14400; # 4h, double the default of 7200s @@ -460,7 +499,7 @@ let in buildPythonPackage { inherit version pname; - disabled = !isPy3k; + disabled = pythonOlder "3.8"; src = bazel-build.python; @@ -493,6 +532,7 @@ in buildPythonPackage { # tensorflow/tools/pip_package/setup.py propagatedBuildInputs = [ absl-py + abseil-cpp astunparse flatbuffers-python gast @@ -529,7 +569,6 @@ in buildPythonPackage { # TEST_PACKAGES in tensorflow/tools/pip_package/setup.py nativeCheckInputs = [ dill - keras portpicker tblib ]; @@ -540,18 +579,15 @@ in buildPythonPackage { hello = tf.constant("Hello, world!") tf.print(hello) - # Fit a simple model to random data - import numpy as np - np.random.seed(0) tf.random.set_seed(0) - model = tf.keras.models.Sequential([ - tf.keras.layers.Dense(1, activation="linear") - ]) - model.compile(optimizer="sgd", loss="mse") - - x = np.random.uniform(size=(1,1)) - y = np.random.uniform(size=(1,)) - model.fit(x, y, epochs=1) + width = 512 + choice = 48 + t_in = tf.Variable(tf.random.uniform(shape=[width])) + with tf.GradientTape() as tape: + t_out = tf.slice(tf.nn.softmax(t_in), [choice], [1]) + diff = tape.gradient(t_out, t_in) + assert(0 < tf.reduce_min(tf.slice(diff, [choice], [1]))) + assert(0 > tf.reduce_max(tf.slice(diff, [1], [choice - 1]))) EOF ''; # Regression test for #77626 removed because not more `tensorflow.contrib`. diff --git a/pkgs/development/python-modules/tensorflow/protobuf_python.patch b/pkgs/development/python-modules/tensorflow/protobuf_python.patch new file mode 100644 index 00000000000..11de26605b4 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/protobuf_python.patch @@ -0,0 +1,19 @@ +diff -Naurd x/tensorflow/workspace2.bzl y/tensorflow/workspace2.bzl +--- x/tensorflow/workspace2.bzl 2023-09-16 20:44:02.266422132 +0000 ++++ y/tensorflow/workspace2.bzl 2023-09-16 20:50:40.370998305 +0000 +@@ -469,6 +469,7 @@ + system_link_files = { + "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", ++ "//third_party/systemlibs:protobuf.python.BUILD": "python/BUILD", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) +diff -Naurd x/third_party/systemlibs/protobuf.python.BUILD y/third_party/systemlibs/protobuf.python.BUILD +--- x/third_party/systemlibs/protobuf.python.BUILD 1970-01-01 00:00:00.000000000 +0000 ++++ y/third_party/systemlibs/protobuf.python.BUILD 2023-09-16 20:49:12.514890584 +0000 +@@ -0,0 +1,4 @@ ++cc_library( ++ name = "proto_api", ++ visibility = ["//visibility:public"] ++) diff --git a/pkgs/development/python-modules/tensorflow/pybind11_protobuf_newer_version.patch b/pkgs/development/python-modules/tensorflow/pybind11_protobuf_newer_version.patch new file mode 100644 index 00000000000..460ada1fb56 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/pybind11_protobuf_newer_version.patch @@ -0,0 +1,16 @@ +diff -Naurd x/tensorflow/workspace2.bzl y/tensorflow/workspace2.bzl +--- x/tensorflow/workspace2.bzl 2023-09-21 12:55:06.104407343 +0000 ++++ y/tensorflow/workspace2.bzl 2023-09-21 12:55:39.732453203 +0000 +@@ -872,9 +872,9 @@ + + tf_http_archive( + name = "pybind11_protobuf", +- urls = tf_mirror_urls("https://github.com/pybind/pybind11_protobuf/archive/80f3440cd8fee124e077e2e47a8a17b78b451363.zip"), +- sha256 = "c7ab64b1ccf9a678694a89035a8c865a693e4e872803778f91f0965c2f281d78", +- strip_prefix = "pybind11_protobuf-80f3440cd8fee124e077e2e47a8a17b78b451363", ++ urls = tf_mirror_urls("https://github.com/pybind/pybind11_protobuf/archive/c8cc30e2495309e3499b7d76033446236d21c837.zip"), ++ sha256 = "0663f73b34c0b4af55003edbb066e62aceff99bfcf12e47ea9a15d6013d81413", ++ strip_prefix = "pybind11_protobuf-c8cc30e2495309e3499b7d76033446236d21c837", + patch_file = [ + "//third_party/pybind11_protobuf:remove_license.patch", + "//third_party/pybind11_protobuf:python_runtime_dep.patch", diff --git a/pkgs/development/python-modules/tensorflow/pybind11_protobuf_python_runtime_dep.patch b/pkgs/development/python-modules/tensorflow/pybind11_protobuf_python_runtime_dep.patch new file mode 100644 index 00000000000..daebaac8689 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/pybind11_protobuf_python_runtime_dep.patch @@ -0,0 +1,30 @@ +diff -Naurd x/tensorflow/workspace2.bzl y/tensorflow/workspace2.bzl +--- x/tensorflow/workspace2.bzl 2023-09-17 22:22:22.029075202 +0000 ++++ y/tensorflow/workspace2.bzl 2023-09-17 22:25:27.501289587 +0000 +@@ -876,7 +876,10 @@ + urls = tf_mirror_urls("https://github.com/pybind/pybind11_protobuf/archive/80f3440cd8fee124e077e2e47a8a17b78b451363.zip"), + sha256 = "c7ab64b1ccf9a678694a89035a8c865a693e4e872803778f91f0965c2f281d78", + strip_prefix = "pybind11_protobuf-80f3440cd8fee124e077e2e47a8a17b78b451363", +- patch_file = ["//third_party/pybind11_protobuf:remove_license.patch"], ++ patch_file = [ ++ "//third_party/pybind11_protobuf:remove_license.patch", ++ "//third_party/pybind11_protobuf:python_runtime_dep.patch", ++ ], + ) + + tf_http_archive( +diff -Naurd x/third_party/pybind11_protobuf/python_runtime_dep.patch y/third_party/pybind11_protobuf/python_runtime_dep.patch +--- x/third_party/pybind11_protobuf/python_runtime_dep.patch 1970-01-01 00:00:00.000000000 +0000 ++++ y/third_party/pybind11_protobuf/python_runtime_dep.patch 2023-09-17 22:23:07.849128180 +0000 +@@ -0,0 +1,11 @@ ++diff -Naurd x/pybind11_protobuf/BUILD y/pybind11_protobuf/BUILD ++--- x/pybind11_protobuf/BUILD 2023-09-17 22:17:19.932725814 +0000 +++++ y/pybind11_protobuf/BUILD 2023-09-17 22:20:44.056961932 +0000 ++@@ -86,6 +86,7 @@ ++ "//visibility:private", ++ ], ++ deps = [ +++ "@org_tensorflow//third_party/python_runtime:headers", ++ "@com_google_absl//absl/container:flat_hash_map", ++ "@com_google_absl//absl/container:flat_hash_set", ++ "@com_google_absl//absl/meta:type_traits", |