WORKING WITH EGI DATAHUB

Contents

WORKING WITH EGI DATAHUB¶

GET EGI access token¶

import requests, json

CLIENT_ID = "f33e824a-078d-497b-b700-25b0df7fc5b7"
CLIENT_SECRET= "B80vPK0LVbYuvwRj0Aexs8y0rKgk5XHwYRRq3BCr33ejj33385bzDVcPmSTUkqA2QjMiwWKJDTxvOou7yVV8EA"
REFRESH_TOKEN= "eyJhbGciOiJub25lIn0.eyJleHAiOjE2NzY1ODM2MTEsImp0aSI6IjFlZGFiOWYwLWJjMDktNDM5Mi05YWRkLTBiN2EwNDk3YmIyMSJ9."
# Retrieving an OIDC token from Check-in

data = {
  'client_id': CLIENT_ID,
  'client_secret': CLIENT_SECRET,
  'refresh_token': REFRESH_TOKEN,
  'scope': 'openid email profile',
  'grant_type': 'refresh_token'
}

response = requests.post('https://aai.egi.eu/oidc/token', data=data, auth=(CLIENT_ID, CLIENT_SECRET))

print(json.dumps(response.json(), indent=2))
rpalmaEGItoken=response.json()['access_token']

{
  "access_token": "eyJraWQiOiJvaWRjIiwiYWxnIjoiUlMyNTYifQ.eyJzdWIiOiI0NDJmOGI0ZGRiYzY0NjM0NGIwMzNmODFkNTM5NzUwMjFjZWZjYWUxNjEwMmNlMzM3ZGFlNmNiN2I0MGNlODg5QGVnaS5ldSIsImF6cCI6ImYzM2U4MjRhLTA3OGQtNDk3Yi1iNzAwLTI1YjBkZjdmYzViNyIsImlzcyI6Imh0dHBzOlwvXC9hYWkuZWdpLmV1XC9vaWRjXC8iLCJleHAiOjE2NDI1MTYwNzcsImlhdCI6MTY0MjUxMjQ3NywianRpIjoiOGNhNzAyNDgtMmM3MS00MThjLWJhOTUtYTg3MzQ3ODhjZGU4In0.B3p-zp8L-hxcVDoGco-kz8iEuwJtRYNW-aaI26YZYg382kaSU261R_Hy_GZ7x9KuMzx3fqxHMWu-z9sd-_mlE1WNTUZlq0i8zxXmxSeN009w4Usi6mXLAdsoNez8texTiQmeQrtk4rTfw9xi7tkj1QM7rIxSQiD_fYZKCCpEpe3TAm19zIuxZTqou9cP5u5vZ1piANwe6vzM8z6noMcutrUNtClGMvO6i-FsC3UpT1ngkkkhn_dagw1hCS1nDGQFuYju05cLRpHZVL9KQotGFZgySdzY1ShMJ-Iy8R24RFsd83bScxQFu22Vl1nHwiC_Bt_Dn9vpQ5P3_Hy7PpzaWg",
  "token_type": "Bearer",
  "refresh_token": "eyJhbGciOiJub25lIn0.eyJleHAiOjE2NzY1ODM2MTEsImp0aSI6IjFlZGFiOWYwLWJjMDktNDM5Mi05YWRkLTBiN2EwNDk3YmIyMSJ9.",
  "expires_in": 3599,
  "scope": "openid profile email",
  "id_token": "eyJraWQiOiJvaWRjIiwiYWxnIjoiUlMyNTYifQ.eyJzdWIiOiI0NDJmOGI0ZGRiYzY0NjM0NGIwMzNmODFkNTM5NzUwMjFjZWZjYWUxNjEwMmNlMzM3ZGFlNmNiN2I0MGNlODg5QGVnaS5ldSIsImF1ZCI6ImYzM2U4MjRhLTA3OGQtNDk3Yi1iNzAwLTI1YjBkZjdmYzViNyIsImtpZCI6Im9pZGMiLCJpc3MiOiJodHRwczpcL1wvYWFpLmVnaS5ldVwvb2lkY1wvIiwiZXhwIjoxNjQyNTEzMDc3LCJpYXQiOjE2NDI1MTI0NzcsIm5vbmNlIjoiMGE1NDk1NzczMzkzMmVjYWZiZWIwOTRiOTA0MzM2ZjAiLCJqdGkiOiJmNWIyZjNhYS1lNjg5LTRmZTItOTRmOS04NzJiM2FiZDBmNTcifQ.YGtI_4pXMF3POH5TC5Hb4kflg5KxmRcNDldy8GrkT8FYYZEALEJ8_fBbK8DyxJC4TDhdhK7GW9xPCbTmH7tMbnm17IMVqi1o7rlMnLF4yikAOkwzc8SyfXF-ns4jYxo3lOV-Ja1nnpsWY9vKU_L-tJSVV7k8_N-62ekITFCcrb8CE9mIrfa_s97WnXbx44TyGoxhuKoJZnNEvvLEghq9_ZU3IZWW4rHc0Ze1fkbBINYDghvYZUtbvzC9myCRL7NsAgQYvSmM8hrSIWuX34yNPMK8lIF8n7c-_MYo9-eXXqzN78dLIz1XRN7M6n514E-M3BjtObpIc9ly1arhkkF5DA"
}

GET DATAHUB_TOKEN¶

import requests, json

headers = {
    'X-Auth-Token': f"egi:"+rpalmaEGItoken,
    'Content-type': 'application/json',
}

data = json.dumps({ 
    'name': 'REST and CDMI access token4', 
    'type': { 
        'accessToken': {} 
    }, 
    'caveats': [ { 
        'type': 'interface', 
        'interface': 'rest' 
    }] 
})
#data = '{ "name": "REST and CDMI access token1", "type": { "accessToken": {} }, "caveats": [ {  "type": "interface",  "interface": "rest" } ] }'


response = requests.post('https://datahub.egi.eu/api/v3/onezone/user/tokens/named', headers=headers, data=data)
print(json.dumps(response.json(), indent=2))
DATAHUB_TOKEN=response.json()['token']

{
  "tokenId": "b5872fbac406cd94c01447374e405e99ch65fd",
  "token": "MDAxY2xvY2F00aW9uIGRhdGFodWIuZWdpLmV1CjAwNmJpZGVudGlmaWVyIDIvbm1kL3Vzci1mMDZmOWZmNDZjOWFjNGFhNjAyNDIwMmVmYmE5Mzc5MWNoOWFjNC9hY3QvYjU4NzJmYmFjNDA2Y2Q5NGMwMTQ00NzM3NGU00MDVlOTljaDY1ZmQKMDAxOWNpZCBpbnRlcmZhY2UgPSByZXN00CjAwMmZzaWduYXR1cmUgPNV8kTXszR5Vt2dF4Saa9cPcYJ7QswPeb9hm02u1MirgK"
}

LIST MY DATAHUB TOKENS¶

import requests, json

headers = {
    'X-Auth-Token': f"egi:"+rpalmaEGItoken,
    'Content-type': 'application/json',
}


response = requests.get('https://datahub.egi.eu/api/v3/onezone/user/tokens/named', headers=headers)
print(json.dumps(response.json(), indent=2))
DATAHUB_TOKEN_ID=response.json()['tokens'][0]

{
  "tokens": [
    "7e7faae7b70ecbf38bc270e2a9cb9944chd88d",
    "0dff794b0579350af80e3d793d954847ch7b18",
    "1fc04058c560432487feb526844c2bd7ch5732",
    "b9b9725ad42e82ebf6f03aed731567e3ch387a",
    "b5872fbac406cd94c01447374e405e99ch65fd",
    "88530800c72623cfac9f3934f4134252chb30c",
    "d918f0ab8c4ce8fcc955af3769f1deffch50ed"
  ]
}

GET TOKEN DETAILS¶

import requests, json

headers = {
    'X-Auth-Token': f"egi:"+rpalmaEGItoken,
    'Content-type': 'application/json',
}


response = requests.get('https://datahub.egi.eu/api/v3/onezone/tokens/named/'+DATAHUB_TOKEN_ID, headers=headers)
print(json.dumps(response.json(), indent=2))

{
  "type": {
    "accessToken": {}
  },
  "token": "MDAxY2xvY2F00aW9uIGRhdGFodWIuZWdpLmV1CjAwNmJpZGVudGlmaWVyIDIvbm1kL3Vzci1mMDZmOWZmNDZjOWFjNGFhNjAyNDIwMmVmYmE5Mzc5MWNoOWFjNC9hY3QvN2U3ZmFhZTdiNzBlY2JmMzhiYzI3MGUyYTljYjk5NDRjaGQ4OGQKMDAxOWNpZCBpbnRlcmZhY2UgPSByZXN00CjAwMmZzaWduYXR1cmUg38z2je500yp4Qozca4zWT2Ok4az00ACBUqxhv2uuU76LIK",
  "subject": {
    "type": "user",
    "id": "f06f9ff46c9ac4aa6024202efba93791ch9ac4"
  },
  "revoked": false,
  "name": "REST and CDMI access token",
  "metadata": {
    "custom": {},
    "creationTime": 1642457185
  },
  "id": "7e7faae7b70ecbf38bc270e2a9cb9944chd88d",
  "caveats": [
    {
      "type": "interface",
      "interface": "rest"
    }
  ]
}

DISPLAY ALL SPACES¶

import requests, json

headers = {
    'X-Auth-Token': DATAHUB_TOKEN
}

response = requests.get('https://datahub.egi.eu/api/v3/onezone/user/effective_spaces/', headers=headers)

print(json.dumps(response.json(), indent=2))

{
  "spaces": [
    "49c131369a88b0b9289b1b578cab7d6ech41aa",
    "51a1b62487f48b27c233746a148b821cch1e86",
    "725f4ab36cb26d0fbfc0c124c775efeechee49"
  ]
}

DISPLAY RELIANCE SPACE DETAILS¶

import requests, json

headers = {
    'X-Auth-Token': DATAHUB_TOKEN
}

response = requests.get('https://datahub.egi.eu/api/v3/onezone/spaces/725f4ab36cb26d0fbfc0c124c775efeechee49/', headers=headers) 
print(json.dumps(response.json(), indent=2))

{
  "spaceId": "725f4ab36cb26d0fbfc0c124c775efeechee49",
  "providers": {
    "3886d8788dbbc4023ec5a9db13c813dech0454": 107374182400
  },
  "name": "Reliance",
  "creator": {
    "type": "user",
    "id": "aa81d77c217c52f3e4c0f79da0fedca0"
  },
  "creationTime": 1638521202
}

DISPLAY RELIANCE SPACE CONTENTS¶

import requests, json

headers = {
    'X-Auth-Token': DATAHUB_TOKEN
}

response = requests.get('https://cesnet-oneprovider-01.datahub.egi.eu/api/v3/oneprovider/files/Reliance/', headers=headers) 
#response = requests.get('https://cesnet-oneprovider-01.datahub.egi.eu/api/v3/oneprovider/files/Reliance/Climate/output', headers=headers) 
print(json.dumps(response.json(), indent=2))

[
  {
    "path": "/Reliance/.ipynb_checkpoints",
    "id": "0000000000520DDF67756964233331663636353734623035346465343163643335613139633038393163643435636835616433233732356634616233366362323664306662666330633132346337373565666565636865653439"
  },
  {
    "path": "/Reliance/Climate",
    "id": "00000000005271F567756964236631353639303132366433336538306631333766333435323162323366316435636832383661233732356634616233366362323664306662666330633132346337373565666565636865653439"
  },
  {
    "path": "/Reliance/Marine Monitoring ",
    "id": "000000000052610967756964233239353265643333313733666433323362373235383232656338356332383138636864343162233732356634616233366362323664306662666330633132346337373565666565636865653439"
  },
  {
    "path": "/Reliance/ROHub-sample-notebooks",
    "id": "000000000052EB6F67756964233531333938313636373738656665643830303839623738323261333765326664636838363065233732356634616233366362323664306662666330633132346337373565666565636865653439"
  },
  {
    "path": "/Reliance/Snapshot-PTF-ML",
    "id": "000000000052B3BF67756964236430623361653332323265336163666135646135326266373734383432336137636836643138233732356634616233366362323664306662666330633132346337373565666565636865653439"
  },
  {
    "path": "/Reliance/Soundscape",
    "id": "000000000052647467756964233565616631613035636165383730646632636236646264613536653637376565636836326431233732356634616233366362323664306662666330633132346337373565666565636865653439"
  }
]

CREATE SHARED LINK¶

import requests, json

headers = {
    'X-Auth-Token': DATAHUB_TOKEN,
    'Content-Type': 'application/json',
}

data = json.dumps({ 
    'name': 'MySharedFileViaAPI.csv',
    'fileId': '000000000052B47167756964236666646366313562336564316562646463623530646666303262333366323161636837386566233732356634616233366362323664306662666330633132346337373565666565636865653439'
    })

response = requests.post('https://cesnet-oneprovider-01.datahub.egi.eu/api/v3/oneprovider/shares', headers=headers, data=data)
print(json.dumps(response.json(), indent=2))
shareIdGenerated=response.json()['shareId']

{
  "shareId": "90c742cf185c699412b371e044c9b1afch1c6a"
}

DISPLAY SHARED LINK DETAILS¶

headers = {
    'X-Auth-Token': DATAHUB_TOKEN
}

# display files in space
response = requests.get('https://cesnet-oneprovider-01.datahub.egi.eu/api/v3/oneprovider/shares/'+shareIdGenerated, headers=headers) 
print(json.dumps(response.json(), indent=2))
publicURL=response.json()['publicUrl']
contentURL= 'https://datahub.egi.eu/api/v3/onezone/shares/data/'+response.json()['rootFileId']+'/content'

{
  "spaceId": "725f4ab36cb26d0fbfc0c124c775efeechee49",
  "shareId": "90c742cf185c699412b371e044c9b1afch1c6a",
  "rootFileId": "00000000007E09EC736861726547756964236666646366313562336564316562646463623530646666303262333366323161636837386566233732356634616233366362323664306662666330633132346337373565666565636865653439233930633734326366313835633639393431326233373165303434633962316166636831633661",
  "publicUrl": "https://datahub.egi.eu/share/90c742cf185c699412b371e044c9b1afch1c6a",
  "publicRestUrl": "https://datahub.egi.eu/api/v3/onezone/shares/90c742cf185c699412b371e044c9b1afch1c6a/public",
  "name": "MySharedFileViaAPI_v2.csv",
  "handleId": null,
  "fileType": "file",
  "description": ""
}

contentURL

'https://datahub.egi.eu/api/v3/onezone/shares/data/00000000007E09EC736861726547756964236666646366313562336564316562646463623530646666303262333366323161636837386566233732356634616233366362323664306662666330633132346337373565666565636865653439233930633734326366313835633639393431326233373165303434633962316166636831633661/content'

previous

Example to use Rohub from Jupyter notebook

next

EOSC-Future demo