Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Annotate Batch Validator] Refactoring #766

Merged
merged 9 commits into from
Aug 22, 2021
Merged

[Annotate Batch Validator] Refactoring #766

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
39bc7b4
adding nextflwo version
mwang87 Aug 20, 2021
8115693
putting it all together
mwang87 Aug 20, 2021
dbd9162
updating
mwang87 Aug 20, 2021
e938995
slight change
mwang87 Aug 20, 2021
71d3db1
adding conversion
mwang87 Aug 21, 2021
7eb6705
working
mwang87 Aug 21, 2021
ac1ce86
adding template workflow
mwang87 Aug 21, 2021
029a4f8
mapping parameters as part of example
mwang87 Aug 21, 2021
0a35776
refining template
mwang87 Aug 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion add-batch-annotated-validator/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@ include ../Makefile.deploytemplate

WORKFLOW_NAME=add-batch-annotated-validator
TOOL_FOLDER_NAME=add-batch-annotated-validator
WORKFLOW_VERSION=release_29
WORKFLOW_VERSION=release_30
WORKFLOW_DESCRIPTION="Validator for Batch Additions"
45 changes: 13 additions & 32 deletions add-batch-annotated-validator/add-batch-annotated-validator/binding.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,45 +19,26 @@
</downloadParams>
</bind>

<bind action="validate_batch" tool="validate_batch">
<inputAsRequirement port="annotation_table" requirement="annotation_table"/>
<productionToOutput port="validation_information" production="validation_information"/>
</bind>

<bind action="convertpklbin" tool="convertpklbin">
<bind action="demangle_spectra" tool="demangle_spectra">
<inputAsRequirement port="spectra" requirement="spectra"/>
<productionToOutput port="convertedspectra" production="convertedspectra"/>
</bind>

<bind action="createlib" tool="AddBatchAnnotated_createlib">
<inputAsRequirement port="spec" requirement="spec"/>
<inputAsRequirement port="flowParams" requirement="flowParams"/>
<inputAsRequirement port="annotation_table_with_id" requirement="annotation_table_with_id"/>
<productionToOutput port="result" production="result"/>
<productionToOutput port="newspectraResult" production="newspectraResult"/>
<productionToOutput port="new_spectra_mgf" production="new_spectra_mgf"/>
<inputAsRequirement port="workflowParameters" requirement="workflowParameters"/>
<productionToOutput port="demangled_spectra" production="demangled_spectra"/>
</bind>

<bind action="formatResult" tool="formatResult">
<bind action="nf_process" tool="nf_process">
<inputAsRequirement port="spectra" requirement="spectra"/>
<inputAsRequirement port="annotation_table" requirement="annotation_table"/>
<inputAsRequirement port="workflowParameters" requirement="workflowParameters"/>
<productionToOutput port="formattedResult" production="formattedResult"/>
<productionToOutput port="workflow_results" production="workflow_results"/>
</bind>

<bind action="end" type="upload">
<url value="{livesearch.upload}"/>
<query name="task" valueRef="@task"/>
<contentQuery name="content"/>
<compression type="zip"/>

<upload port="newspectraResult" type="file">
<query name="resource" value="newspectraResult"/>
</upload>

<upload port="formattedResult" type="file">
<query name="resource" value="formattedResult"/>
</upload>


<url value="{livesearch.upload}"/>
<query name="task" valueRef="@task"/>
<contentQuery name="content"/>
<compression type="zip"/>
<upload port="workflow_results" type="folder">
<query name="resource" value="workflow_results"/>
</upload>
</bind>
</binding>
21 changes: 17 additions & 4 deletions add-batch-annotated-validator/add-batch-annotated-validator/flow.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,14 @@
<output port="flowParams" object="workflowParameters"/>
</action>

<collection name="validation_information"/>
<collection name="demangled_spectra"/>
<action name="demangle_spectra">
<input port="spectra" collection="spectra"/>
<input port="workflowParameters" object="workflowParameters"/>
<output port="demangled_spectra" collection="demangled_spectra"/>
</action>

<!-- <collection name="validation_information"/>
<collection name="control_flow_dependency"/>

<action name="validate_batch">
Expand Down Expand Up @@ -44,11 +51,17 @@
<input port="annotation_table" object="annotation_table"/>
<input port="workflowParameters" object="workflowParameters"/>
<output port="formattedResult" object="formattedResult"/>
</action> -->

<collection name="workflow_results"/>
<action name="nf_process">
<input port="spectra" collection="spectra"/>
<input port="annotation_table" object="annotation_table"/>
<input port="workflowParameters" object="workflowParameters"/>
<output port="workflow_results" collection="workflow_results"/>
</action>

<action name="end">
<input port="newspectraResult" object="newspectraResult"/>
<input port="validation_information" collection="validation_information"/>
<input port="formattedResult" collection="formattedResult"/>
<input port="workflow_results" collection="workflow_results"/>
</action>
</flow>
4 changes: 2 additions & 2 deletions add-batch-annotated-validator/add-batch-annotated-validator/result.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

<block id="new_annotations" type="table">
<data>
<source type="file" name="newspectraResult/"/>
<source type="file" name="workflow_results/new_results.tsv"/>
<parsers>
<parser type="sortedTabular" sortBy="CompoundName" operator="ascending"/>
</parsers>
Expand Down Expand Up @@ -49,7 +49,7 @@

<block id="view_annotation" type="table">
<data>
<source name="formattedResult/formatted_annotation.tsv" type="file"/>
<source name="workflow_results/formattedresult.tsv" type="file"/>
<parsers>
<parser type="genericTabular"/>
</parsers>
Expand Down
100 changes: 37 additions & 63 deletions add-batch-annotated-validator/add-batch-annotated-validator/tool.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,87 +1,61 @@
<toolset>
<pathSet base="anaconda3/bin">
<toolPath tool="validate_batch" path="python3.5"/>
<!-- Demangling -->
<pathSet base="miniconda3_gamma/envs/msql2/bin">
<toolPath tool="demangle_spectra" path="python3"/>
</pathSet>

<pathSet base="$base">
<pathVar name="validate_batch.script" path="validate_batch.py"/>
<pathVar name="demangle.script" path="demangle_collection.py"/>
</pathSet>

<tool name="validate_batch">
<require name="annotation_table" type="file"/>
<produce name="validation_information" type="folder"/>

<execution env="binary" argConvention="adhoc">
<arg pathRef="validate_batch.script"/>
<arg valueRef="annotation_table"/>
<arg valueRef="validation_information"/>
</execution>
</tool>

<pathSet base="$base">
<pathVar name="convertexe.file" path="convert"/>
<toolPath tool="convertpklbin" path="convert_wrapper.py"/>
</pathSet>

<!-- Spectral library manipulation tool specifications -->
<tool name="convertpklbin">
<tool name="demangle_spectra">
<require name="spectra" type="folder"/>
<produce name="convertedspectra" type="folder"/>
<require name="workflowParameters" type="file"/>
<produce name="demangled_spectra" type="folder"/>

<execution env="binary" argConvention="adhoc">
<arg valueRef="spectra"/>
<arg valueRef="convertedspectra"/>
<arg pathRef="convertexe.file"/>
<arg pathRef="demangle.script"/>
<arg option="p" valueRef="workflowParameters"/>
<arg option="-input_mangled_prefix" value="spec"/>
<arg option="i" valueRef="spectra"/>
<arg option="o" valueRef="demangled_spectra"/>
</execution>
</tool>

<!-- Spectral library manipulation tool specifications -->
<pathSet base="anaconda3/bin">
<toolPath tool="AddBatchAnnotated_createlib" path="python3.5"/>
</pathSet>
<pathSet base="$base">
<pathVar name="AddBatchAnnotated_createlib.exe" path="main_execmodule"/>
<pathVar name="AddBatchAnnotated_createlib.script" path="create_lib_wrapper.py"/>
<pathSet base="miniconda3_gamma/envs/msql2/bin">
<toolPath tool="nf_process" path="python3"/>
<pathVar name="python_runtime" path="python3"/>
</pathSet>

<tool name="AddBatchAnnotated_createlib">
<require name="spec" type="folder"/>
<require name="flowParams" type="file"/>
<require name="annotation_table_with_id" type="file"/>
<produce name="result" type="file" naming="explicit" extension="out"/>
<produce name="newspectraResult" type="file" naming="explicit" extension="out"/>
<produce name="new_spectra_mgf" type="file" naming="explicit" extension="mgf"/>

<execution env="binary" argConvention="adhoc">
<arg pathRef="AddBatchAnnotated_createlib.script"/>
<arg valueRef="flowParams"/>
<arg valueRef="spec"/>
<arg valueRef="annotation_table_with_id"/>
<arg valueRef="result"/>
<arg valueRef="newspectraResult"/>
<arg valueRef="new_spectra_mgf"/>
<arg pathRef="AddBatchAnnotated_createlib.exe"/>
</execution>
</tool>


<pathSet base="anaconda3/bin">
<toolPath tool="formatResult" path="python3.5"/>
<pathSet base="miniconda3_gamma/bin">
<pathVar name="conda_activate_path" path="activate"/>
</pathSet>

<pathSet base="$base">
<pathVar name="formatResult.script" path="formatresults.py"/>
<pathVar name="nextflow_wrapper.script" path="nextflow_wrapper.py"/>
<pathVar name="nextflow.script" path="workflow.nf"/>
</pathSet>

<tool name="formatResult">
<require name="annotation_table" type="file"/>
<tool name="nf_process">
<require name="spectra" type="folder"/>
<require name="workflowParameters" type="file"/>
<produce name="formattedResult" type="file" naming="fixed" value="formatted_annotation.tsv"/>
<require name="annotation_table" type="file"/>
<produce name="workflow_results" type="folder"/>

<execution env="binary" argConvention="adhoc">
<arg pathRef="formatResult.script"/>
<arg valueRef="annotation_table"/>
<arg pathRef="nextflow_wrapper.script"/>
<arg valueRef="workflowParameters"/>
<arg valueRef="formattedResult"/>
<arg pathRef="nextflow.script"/>
<arg pathRef="conda_activate_path"/>
<arg value="msql2"/>

<arg option="-metricoutput" valueRef="workflow_results"/>

<!-- This portion is currently hardcoded to the nextflow workflow. -->

<arg option="-newparameters" value="publishdir:workflow_results"/>
<arg option="-newparameters" value="inputspectra:spectra"/>
<arg option="-newparameters" value="annotation_table:annotation_table/*"/>
<arg option="-newparameters" value="workflowParameters:workflowParameters/*"/>
</execution>
</tool>

Expand Down
2 changes: 2 additions & 0 deletions add-batch-annotated-validator/test-integration-workflow/test_tasks.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
task_id,description,regressioncountviews
89986d09b01d4ee7966a80a2382c319d,chooser test,
3 changes: 3 additions & 0 deletions add-batch-annotated-validator/test/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
test_workflow:
cd ../tools/add-batch-annotated-validator && nextflow run workflow.nf \
--annotation_table="../../test/data/MSMS-CHOOSER-7c0df335-download_table-main.tsv"
1 change: 1 addition & 0 deletions add-batch-annotated-validator/tools/add-batch-annotated-validator/bin/convert
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../../../shared_code/bin/convert
25 changes: 25 additions & 0 deletions add-batch-annotated-validator/tools/add-batch-annotated-validator/bin/convert_wrapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#!/usr/bin/python

import sys
import getopt
import os

def main():

input_folder_path = sys.argv[1]
output_folder_path = sys.argv[2]
convert_path = sys.argv[3]

files = os.listdir(input_folder_path)

for input_file in files:
fileName, fileExtension = os.path.splitext(input_file)
if input_file.find("mzXML") != -1 or input_file.find("mzML") != -1:
cmd = convert_path + " " + input_folder_path + "/" + input_file + " " + output_folder_path + "/" + fileName + ".pklbin"
else:
cmd = "cp " + input_folder_path + "/" + input_file + " " + output_folder_path + "/" + input_file
print(cmd)
os.system(cmd)

if __name__ == "__main__":
main()
0 ...annotated-validator/create_lib_wrapper.py → ...tated-validator/bin/create_lib_wrapper.py
View file
Open in desktop
File renamed without changes.
0 ...atch-annotated-validator/formatresults.py → ...-annotated-validator/bin/formatresults.py
View file
Open in desktop
File renamed without changes.
0 ...batch-annotated-validator/main_execmodule → ...h-annotated-validator/bin/main_execmodule
View file
Open in desktop
File renamed without changes.
1 change: 1 addition & 0 deletions add-batch-annotated-validator/tools/add-batch-annotated-validator/bin/shared_code
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../../../shared_code/
0 ...tch-annotated-validator/validate_batch.py → ...annotated-validator/bin/validate_batch.py
View file
Open in desktop
File renamed without changes.
1 change: 0 additions & 1 deletion add-batch-annotated-validator/tools/add-batch-annotated-validator/convert
View file
Open in desktop

This file was deleted.

56 changes: 0 additions & 56 deletions add-batch-annotated-validator/tools/add-batch-annotated-validator/convert_wrapper.py
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions add-batch-annotated-validator/tools/add-batch-annotated-validator/demangle_collection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../../shared_code/demangle_collection.py
1 change: 1 addition & 0 deletions add-batch-annotated-validator/tools/add-batch-annotated-validator/ming_fileio_library.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../../shared_code/ming_fileio_library.py
1 change: 1 addition & 0 deletions add-batch-annotated-validator/tools/add-batch-annotated-validator/ming_proteosafe_library.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../../shared_code/ming_proteosafe_library.py
1 change: 1 addition & 0 deletions add-batch-annotated-validator/tools/add-batch-annotated-validator/nextflow_wrapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../../shared_code/nextflow_wrapper.py
1 change: 0 additions & 1 deletion add-batch-annotated-validator/tools/add-batch-annotated-validator/shared_code
View file
Open in desktop

This file was deleted.

Loading