fugue

• fugue.bag
  • fugue.bag.array_bag
    • ArrayBag
      • ArrayBag.as_array()
      • ArrayBag.count()
      • ArrayBag.empty
      • ArrayBag.head()
      • ArrayBag.native
      • ArrayBag.peek()
  • fugue.bag.bag
    • Bag
      • Bag.as_array()
      • Bag.as_local()
      • Bag.as_local_bounded()
      • Bag.head()
      • Bag.peek()
    • BagDisplay
      • BagDisplay.bg
      • BagDisplay.show()
    • LocalBag
      • LocalBag.is_local
      • LocalBag.num_partitions
    • LocalBoundedBag
      • LocalBoundedBag.as_local_bounded()
      • LocalBoundedBag.is_bounded
• fugue.collections
  • fugue.collections.partition
    • BagPartitionCursor
    • DatasetPartitionCursor
      • DatasetPartitionCursor.item
      • DatasetPartitionCursor.partition_no
      • DatasetPartitionCursor.physical_partition_no
      • DatasetPartitionCursor.set()
      • DatasetPartitionCursor.slice_no
    • PartitionCursor
      • PartitionCursor.key_schema
      • PartitionCursor.key_value_array
      • PartitionCursor.key_value_dict
      • PartitionCursor.row
      • PartitionCursor.row_schema
      • PartitionCursor.set()
    • PartitionSpec
      • PartitionSpec.algo
      • PartitionSpec.empty
      • PartitionSpec.get_cursor()
      • PartitionSpec.get_key_schema()
      • PartitionSpec.get_num_partitions()
      • PartitionSpec.get_partitioner()
      • PartitionSpec.get_sorts()
      • PartitionSpec.jsondict
      • PartitionSpec.num_partitions
      • PartitionSpec.partition_by
      • PartitionSpec.presort
      • PartitionSpec.presort_expr
    • parse_presort_exp()
  • fugue.collections.sql
    • StructuredRawSQL
      • StructuredRawSQL.construct()
      • StructuredRawSQL.dialect
      • StructuredRawSQL.from_expr()
    • TempTableName
  • fugue.collections.yielded
    • PhysicalYielded
      • PhysicalYielded.is_set
      • PhysicalYielded.name
      • PhysicalYielded.set_value()
      • PhysicalYielded.storage_type
    • Yielded
      • Yielded.is_set
• fugue.column
  • fugue.column.expressions
    • ColumnExpr
      • ColumnExpr.alias()
      • ColumnExpr.as_name
      • ColumnExpr.as_type
      • ColumnExpr.body_str
      • ColumnExpr.cast()
      • ColumnExpr.infer_alias()
      • ColumnExpr.infer_type()
      • ColumnExpr.is_null()
      • ColumnExpr.name
      • ColumnExpr.not_null()
      • ColumnExpr.output_name
    • all_cols()
    • col()
    • function()
    • lit()
    • null()
  • fugue.column.functions
    • avg()
    • coalesce()
    • count()
    • count_distinct()
    • first()
    • is_agg()
    • last()
    • max()
    • min()
    • sum()
  • fugue.column.sql
    • SQLExpressionGenerator
      • SQLExpressionGenerator.add_func_handler()
      • SQLExpressionGenerator.correct_select_schema()
      • SQLExpressionGenerator.generate()
      • SQLExpressionGenerator.select()
      • SQLExpressionGenerator.type_to_expr()
      • SQLExpressionGenerator.where()
    • SelectColumns
      • SelectColumns.agg_funcs
      • SelectColumns.all_cols
      • SelectColumns.assert_all_with_names()
      • SelectColumns.assert_no_agg()
      • SelectColumns.assert_no_wildcard()
      • SelectColumns.group_keys
      • SelectColumns.has_agg
      • SelectColumns.has_literals
      • SelectColumns.is_distinct
      • SelectColumns.literals
      • SelectColumns.non_agg_funcs
      • SelectColumns.replace_wildcard()
      • SelectColumns.simple
      • SelectColumns.simple_cols
• fugue.dataframe
  • fugue.dataframe.api
    • get_native_as_df()
    • normalize_column_names()
  • fugue.dataframe.array_dataframe
    • ArrayDataFrame
      • ArrayDataFrame.alter_columns()
      • ArrayDataFrame.as_array()
      • ArrayDataFrame.as_array_iterable()
      • ArrayDataFrame.count()
      • ArrayDataFrame.empty
      • ArrayDataFrame.head()
      • ArrayDataFrame.native
      • ArrayDataFrame.peek_array()
      • ArrayDataFrame.rename()
  • fugue.dataframe.arrow_dataframe
    • ArrowDataFrame
      • ArrowDataFrame.alter_columns()
      • ArrowDataFrame.as_array()
      • ArrowDataFrame.as_array_iterable()
      • ArrowDataFrame.as_arrow()
      • ArrowDataFrame.as_pandas()
      • ArrowDataFrame.count()
      • ArrowDataFrame.empty
      • ArrowDataFrame.head()
      • ArrowDataFrame.native
      • ArrowDataFrame.native_as_df()
      • ArrowDataFrame.peek_array()
      • ArrowDataFrame.peek_dict()
      • ArrowDataFrame.rename()
  • fugue.dataframe.dataframe
    • DataFrame
      • DataFrame.alter_columns()
      • DataFrame.as_array()
      • DataFrame.as_array_iterable()
      • DataFrame.as_arrow()
      • DataFrame.as_dict_iterable()
      • DataFrame.as_local()
      • DataFrame.as_local_bounded()
      • DataFrame.as_pandas()
      • DataFrame.columns
      • DataFrame.drop()
      • DataFrame.get_info_str()
      • DataFrame.head()
      • DataFrame.native_as_df()
      • DataFrame.peek_array()
      • DataFrame.peek_dict()
      • DataFrame.rename()
      • DataFrame.schema
      • DataFrame.schema_discovered
    • DataFrameDisplay
      • DataFrameDisplay.df
      • DataFrameDisplay.show()
    • LocalBoundedDataFrame
      • LocalBoundedDataFrame.as_local_bounded()
      • LocalBoundedDataFrame.is_bounded
    • LocalDataFrame
      • LocalDataFrame.is_local
      • LocalDataFrame.native_as_df()
      • LocalDataFrame.num_partitions
    • LocalUnboundedDataFrame
      • LocalUnboundedDataFrame.as_local()
      • LocalUnboundedDataFrame.count()
      • LocalUnboundedDataFrame.is_bounded
    • YieldedDataFrame
      • YieldedDataFrame.is_set
      • YieldedDataFrame.result
      • YieldedDataFrame.set_value()
    • as_fugue_df()
  • fugue.dataframe.dataframe_iterable_dataframe
    • IterableArrowDataFrame
    • IterablePandasDataFrame
      • IterablePandasDataFrame.as_local_bounded()
    • LocalDataFrameIterableDataFrame
      • LocalDataFrameIterableDataFrame.alter_columns()
      • LocalDataFrameIterableDataFrame.as_array()
      • LocalDataFrameIterableDataFrame.as_array_iterable()
      • LocalDataFrameIterableDataFrame.as_arrow()
      • LocalDataFrameIterableDataFrame.as_local_bounded()
      • LocalDataFrameIterableDataFrame.as_pandas()
      • LocalDataFrameIterableDataFrame.empty
      • LocalDataFrameIterableDataFrame.head()
      • LocalDataFrameIterableDataFrame.native
      • LocalDataFrameIterableDataFrame.peek_array()
      • LocalDataFrameIterableDataFrame.rename()
  • fugue.dataframe.dataframes
    • DataFrames
      • DataFrames.convert()
      • DataFrames.has_key
  • fugue.dataframe.function_wrapper
    • DataFrameFunctionWrapper
      • DataFrameFunctionWrapper.get_format_hint()
      • DataFrameFunctionWrapper.need_output_schema
      • DataFrameFunctionWrapper.run()
    • DataFrameParam
      • DataFrameParam.count()
      • DataFrameParam.to_input_data()
      • DataFrameParam.to_output_df()
    • LocalDataFrameParam
      • LocalDataFrameParam.count()
      • LocalDataFrameParam.to_input_data()
      • LocalDataFrameParam.to_output_df()
  • fugue.dataframe.iterable_dataframe
    • IterableDataFrame
      • IterableDataFrame.alter_columns()
      • IterableDataFrame.as_array()
      • IterableDataFrame.as_array_iterable()
      • IterableDataFrame.as_local_bounded()
      • IterableDataFrame.empty
      • IterableDataFrame.head()
      • IterableDataFrame.native
      • IterableDataFrame.peek_array()
      • IterableDataFrame.rename()
  • fugue.dataframe.pandas_dataframe
    • PandasDataFrame
      • PandasDataFrame.alter_columns()
      • PandasDataFrame.as_array()
      • PandasDataFrame.as_array_iterable()
      • PandasDataFrame.as_pandas()
      • PandasDataFrame.count()
      • PandasDataFrame.empty
      • PandasDataFrame.head()
      • PandasDataFrame.native
      • PandasDataFrame.native_as_df()
      • PandasDataFrame.peek_array()
      • PandasDataFrame.rename()
  • fugue.dataframe.utils
    • deserialize_df()
    • get_join_schemas()
    • pickle_df()
    • serialize_df()
    • unpickle_df()
• fugue.dataset
  • fugue.dataset.api
    • show()
  • fugue.dataset.dataset
    • Dataset
      • Dataset.assert_not_empty()
      • Dataset.count()
      • Dataset.empty
      • Dataset.has_metadata
      • Dataset.is_bounded
      • Dataset.is_local
      • Dataset.metadata
      • Dataset.native
      • Dataset.num_partitions
      • Dataset.reset_metadata()
      • Dataset.show()
    • DatasetDisplay
      • DatasetDisplay.repr()
      • DatasetDisplay.repr_html()
      • DatasetDisplay.show()
• fugue.execution
  • fugue.execution.api
    • aggregate()
    • anti_join()
    • assign()
    • broadcast()
    • clear_global_engine()
    • cross_join()
    • distinct()
    • dropna()
    • engine_context()
    • fillna()
    • filter()
    • full_outer_join()
    • get_context_engine()
    • get_current_conf()
    • get_current_parallelism()
    • inner_join()
    • intersect()
    • join()
    • left_outer_join()
    • load()
    • persist()
    • repartition()
    • right_outer_join()
    • run_engine_function()
    • sample()
    • save()
    • select()
    • semi_join()
    • set_global_engine()
    • subtract()
    • take()
    • union()
  • fugue.execution.execution_engine
    • EngineFacet
      • EngineFacet.conf
      • EngineFacet.execution_engine
      • EngineFacet.execution_engine_constraint
      • EngineFacet.log
      • EngineFacet.to_df()
    • ExecutionEngine
      • ExecutionEngine.aggregate()
      • ExecutionEngine.as_context()
      • ExecutionEngine.assign()
      • ExecutionEngine.broadcast()
      • ExecutionEngine.comap()
      • ExecutionEngine.conf
      • ExecutionEngine.convert_yield_dataframe()
      • ExecutionEngine.create_default_map_engine()
      • ExecutionEngine.create_default_sql_engine()
      • ExecutionEngine.distinct()
      • ExecutionEngine.dropna()
      • ExecutionEngine.fillna()
      • ExecutionEngine.filter()
      • ExecutionEngine.fs
      • ExecutionEngine.get_current_parallelism()
      • ExecutionEngine.in_context
      • ExecutionEngine.intersect()
      • ExecutionEngine.is_global
      • ExecutionEngine.join()
      • ExecutionEngine.load_df()
      • ExecutionEngine.load_yielded()
      • ExecutionEngine.map_engine
      • ExecutionEngine.on_enter_context()
      • ExecutionEngine.on_exit_context()
      • ExecutionEngine.persist()
      • ExecutionEngine.repartition()
      • ExecutionEngine.sample()
      • ExecutionEngine.save_df()
      • ExecutionEngine.select()
      • ExecutionEngine.set_global()
      • ExecutionEngine.set_sql_engine()
      • ExecutionEngine.sql_engine
      • ExecutionEngine.stop()
      • ExecutionEngine.stop_engine()
      • ExecutionEngine.subtract()
      • ExecutionEngine.take()
      • ExecutionEngine.union()
      • ExecutionEngine.zip()
      • ExecutionEngine.zip_all()
    • ExecutionEngineParam
      • ExecutionEngineParam.to_input()
    • FugueEngineBase
      • FugueEngineBase.conf
      • FugueEngineBase.is_distributed
      • FugueEngineBase.log
      • FugueEngineBase.to_df()
    • MapEngine
      • MapEngine.map_bag()
      • MapEngine.map_dataframe()
    • SQLEngine
      • SQLEngine.dialect
      • SQLEngine.encode()
      • SQLEngine.encode_name()
      • SQLEngine.load_table()
      • SQLEngine.save_table()
      • SQLEngine.select()
      • SQLEngine.table_exists()
  • fugue.execution.factory
    • is_pandas_or()
    • make_execution_engine()
    • make_sql_engine()
    • register_default_execution_engine()
    • register_default_sql_engine()
    • register_execution_engine()
    • register_sql_engine()
    • try_get_context_execution_engine()
  • fugue.execution.native_execution_engine
    • NativeExecutionEngine
      • NativeExecutionEngine.broadcast()
      • NativeExecutionEngine.create_default_map_engine()
      • NativeExecutionEngine.create_default_sql_engine()
      • NativeExecutionEngine.distinct()
      • NativeExecutionEngine.dropna()
      • NativeExecutionEngine.fillna()
      • NativeExecutionEngine.fs
      • NativeExecutionEngine.get_current_parallelism()
      • NativeExecutionEngine.intersect()
      • NativeExecutionEngine.is_distributed
      • NativeExecutionEngine.join()
      • NativeExecutionEngine.load_df()
      • NativeExecutionEngine.log
      • NativeExecutionEngine.persist()
      • NativeExecutionEngine.pl_utils
      • NativeExecutionEngine.repartition()
      • NativeExecutionEngine.sample()
      • NativeExecutionEngine.save_df()
      • NativeExecutionEngine.subtract()
      • NativeExecutionEngine.take()
      • NativeExecutionEngine.to_df()
      • NativeExecutionEngine.union()
    • PandasMapEngine
      • PandasMapEngine.execution_engine_constraint
      • PandasMapEngine.is_distributed
      • PandasMapEngine.map_dataframe()
      • PandasMapEngine.to_df()
    • QPDPandasEngine
      • QPDPandasEngine.dialect
      • QPDPandasEngine.is_distributed
      • QPDPandasEngine.select()
      • QPDPandasEngine.to_df()
• fugue.extensions
  • fugue.extensions.creator
    • fugue.extensions.creator.convert
      • creator()
      • register_creator()
    • fugue.extensions.creator.creator
      • Creator
  • fugue.extensions.outputter
    • fugue.extensions.outputter.convert
      • outputter()
      • register_outputter()
    • fugue.extensions.outputter.outputter
      • Outputter
  • fugue.extensions.processor
    • fugue.extensions.processor.convert
      • processor()
      • register_processor()
    • fugue.extensions.processor.processor
      • Processor
  • fugue.extensions.transformer
    • fugue.extensions.transformer.constants
    • fugue.extensions.transformer.convert
      • cotransformer()
      • output_cotransformer()
      • output_transformer()
      • register_output_transformer()
      • register_transformer()
      • transformer()
    • fugue.extensions.transformer.transformer
      • CoTransformer
      • OutputCoTransformer
      • OutputTransformer
      • Transformer
  • fugue.extensions.context
    • ExtensionContext
      • ExtensionContext.callback
      • ExtensionContext.cursor
      • ExtensionContext.execution_engine
      • ExtensionContext.has_callback
      • ExtensionContext.key_schema
      • ExtensionContext.output_schema
      • ExtensionContext.params
      • ExtensionContext.partition_spec
      • ExtensionContext.rpc_server
      • ExtensionContext.validate_on_compile()
      • ExtensionContext.validate_on_runtime()
      • ExtensionContext.validation_rules
      • ExtensionContext.workflow_conf
• fugue.rpc
  • fugue.rpc.base
    • EmptyRPCHandler
    • NativeRPCClient
    • NativeRPCServer
      • NativeRPCServer.make_client()
      • NativeRPCServer.start_server()
      • NativeRPCServer.stop_server()
    • RPCClient
    • RPCFunc
    • RPCHandler
      • RPCHandler.running
      • RPCHandler.start()
      • RPCHandler.start_handler()
      • RPCHandler.stop()
      • RPCHandler.stop_handler()
    • RPCServer
      • RPCServer.conf
      • RPCServer.invoke()
      • RPCServer.make_client()
      • RPCServer.register()
      • RPCServer.start_handler()
      • RPCServer.start_server()
      • RPCServer.stop_handler()
      • RPCServer.stop_server()
    • make_rpc_server()
    • to_rpc_handler()
  • fugue.rpc.flask
    • FlaskRPCClient
    • FlaskRPCServer
      • FlaskRPCServer.make_client()
      • FlaskRPCServer.start_server()
      • FlaskRPCServer.stop_server()
• fugue.sql
  • fugue.sql.api
    • fugue_sql()
    • fugue_sql_flow()
  • fugue.sql.workflow
    • FugueSQLWorkflow
      • FugueSQLWorkflow.sql_vars
• fugue.workflow
  • fugue.workflow.api
    • out_transform()
    • raw_sql()
    • transform()
  • fugue.workflow.input
    • register_raw_df_type()
  • fugue.workflow.module
    • module()
  • fugue.workflow.workflow
    • FugueWorkflow
      • FugueWorkflow.add()
      • FugueWorkflow.assert_eq()
      • FugueWorkflow.assert_not_eq()
      • FugueWorkflow.conf
      • FugueWorkflow.create()
      • FugueWorkflow.create_data()
      • FugueWorkflow.df()
      • FugueWorkflow.get_result()
      • FugueWorkflow.intersect()
      • FugueWorkflow.join()
      • FugueWorkflow.last_df
      • FugueWorkflow.load()
      • FugueWorkflow.out_transform()
      • FugueWorkflow.output()
      • FugueWorkflow.process()
      • FugueWorkflow.run()
      • FugueWorkflow.select()
      • FugueWorkflow.set_op()
      • FugueWorkflow.show()
      • FugueWorkflow.spec_uuid()
      • FugueWorkflow.subtract()
      • FugueWorkflow.transform()
      • FugueWorkflow.union()
      • FugueWorkflow.yields
      • FugueWorkflow.zip()
    • FugueWorkflowResult
      • FugueWorkflowResult.yields
    • WorkflowDataFrame
      • WorkflowDataFrame.aggregate()
      • WorkflowDataFrame.alter_columns()
      • WorkflowDataFrame.anti_join()
      • WorkflowDataFrame.as_array()
      • WorkflowDataFrame.as_array_iterable()
      • WorkflowDataFrame.as_local()
      • WorkflowDataFrame.as_local_bounded()
      • WorkflowDataFrame.assert_eq()
      • WorkflowDataFrame.assert_not_eq()
      • WorkflowDataFrame.assign()
      • WorkflowDataFrame.broadcast()
      • WorkflowDataFrame.checkpoint()
      • WorkflowDataFrame.compute()
      • WorkflowDataFrame.count()
      • WorkflowDataFrame.cross_join()
      • WorkflowDataFrame.deterministic_checkpoint()
      • WorkflowDataFrame.distinct()
      • WorkflowDataFrame.drop()
      • WorkflowDataFrame.dropna()
      • WorkflowDataFrame.empty
      • WorkflowDataFrame.fillna()
      • WorkflowDataFrame.filter()
      • WorkflowDataFrame.full_outer_join()
      • WorkflowDataFrame.head()
      • WorkflowDataFrame.inner_join()
      • WorkflowDataFrame.intersect()
      • WorkflowDataFrame.is_bounded
      • WorkflowDataFrame.is_local
      • WorkflowDataFrame.join()
      • WorkflowDataFrame.left_anti_join()
      • WorkflowDataFrame.left_outer_join()
      • WorkflowDataFrame.left_semi_join()
      • WorkflowDataFrame.name
      • WorkflowDataFrame.native
      • WorkflowDataFrame.native_as_df()
      • WorkflowDataFrame.num_partitions
      • WorkflowDataFrame.out_transform()
      • WorkflowDataFrame.output()
      • WorkflowDataFrame.partition()
      • WorkflowDataFrame.partition_by()
      • WorkflowDataFrame.partition_spec
      • WorkflowDataFrame.peek_array()
      • WorkflowDataFrame.per_partition_by()
      • WorkflowDataFrame.per_row()
      • WorkflowDataFrame.persist()
      • WorkflowDataFrame.process()
      • WorkflowDataFrame.rename()
      • WorkflowDataFrame.result
      • WorkflowDataFrame.right_outer_join()
      • WorkflowDataFrame.sample()
      • WorkflowDataFrame.save()
      • WorkflowDataFrame.save_and_use()
      • WorkflowDataFrame.schema
      • WorkflowDataFrame.select()
      • WorkflowDataFrame.semi_join()
      • WorkflowDataFrame.show()
      • WorkflowDataFrame.spec_uuid()
      • WorkflowDataFrame.strong_checkpoint()
      • WorkflowDataFrame.subtract()
      • WorkflowDataFrame.take()
      • WorkflowDataFrame.transform()
      • WorkflowDataFrame.union()
      • WorkflowDataFrame.weak_checkpoint()
      • WorkflowDataFrame.workflow
      • WorkflowDataFrame.yield_dataframe_as()
      • WorkflowDataFrame.yield_file_as()
      • WorkflowDataFrame.yield_table_as()
      • WorkflowDataFrame.zip()
    • WorkflowDataFrames
      • WorkflowDataFrames.workflow

fugue.api

fugue.constants

fugue.constants.register_global_conf(conf, on_dup=0)

Register global Fugue configs that can be picked up by any Fugue execution engines as the base configs.

Parameters

• conf (Dict[str, Any]) – the config dictionary

• on_dup (int) – see triad.collections.dict.ParamDict.update() , defaults to ParamDict.OVERWRITE

Return type

None

Note

When using ParamDict.THROW or on_dup, it’s transactional. If any key in conf is already in global config and the value is different from the new value, then ValueError will be thrown.

Examples

from fugue import register_global_conf, NativeExecutionEngine

register_global_conf({"my.value",1})

engine = NativeExecutionEngine()
assert 1 == engine.conf["my.value"]

engine = NativeExecutionEngine({"my.value",2})
assert 2 == engine.conf["my.value"]

fugue.dev

All modeuls for developing and extending Fugue

fugue.exceptions

exception fugue.exceptions.FugueBug

Bases: FugueError

Fugue internal bug

exception fugue.exceptions.FugueDataFrameError

Bases: FugueError

Fugue dataframe related error

exception fugue.exceptions.FugueDataFrameInitError

Bases: FugueDataFrameError

Fugue dataframe initialization error

exception fugue.exceptions.FugueDataFrameOperationError

Bases: FugueDataFrameError

Fugue dataframe invalid operation

exception fugue.exceptions.FugueDatasetEmptyError

Bases: FugueDataFrameError

Fugue dataframe is empty

exception fugue.exceptions.FugueError

Bases: Exception

Fugue exceptions

exception fugue.exceptions.FugueInterfacelessError

Bases: FugueWorkflowCompileError

Fugue interfaceless exceptions

exception fugue.exceptions.FugueInvalidOperation

Bases: FugueError

Invalid operation on the Fugue framework

exception fugue.exceptions.FuguePluginsRegistrationError

Bases: FugueError

Fugue plugins registration error

exception fugue.exceptions.FugueSQLError

Bases: FugueWorkflowCompileError

Fugue SQL error

exception fugue.exceptions.FugueSQLRuntimeError

Bases: FugueWorkflowRuntimeError

Fugue SQL runtime error

exception fugue.exceptions.FugueSQLSyntaxError

Bases: FugueSQLError

Fugue SQL syntax error

exception fugue.exceptions.FugueWorkflowCompileError

Bases: FugueWorkflowError

Fugue workflow compile time error

exception fugue.exceptions.FugueWorkflowCompileValidationError

Bases: FugueWorkflowCompileError

Fugue workflow compile time validation error

exception fugue.exceptions.FugueWorkflowError

Bases: FugueError

Fugue workflow exceptions

exception fugue.exceptions.FugueWorkflowRuntimeError

Bases: FugueWorkflowError

Fugue workflow compile time error

exception fugue.exceptions.FugueWorkflowRuntimeValidationError

Bases: FugueWorkflowRuntimeError

Fugue workflow runtime validation error

fugue.plugins

fugue.registry